public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-04-13 09:42:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merging all four node servers into a single one on port 8008. Updating HAProxy on both consul and docker-compose.

Browse source
This commit is contained in:
Samuel Clay 2021-03-16 19:34:11 -04:00
parent 3f0b2ef366
commit 2906e92499
17 changed files with 578 additions and 638 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
ansible/roles/node/tasks/main.yml
View file

@ -2,10 +2,10 @@
- name: Start node docker containers
become: yes
docker_container:
name: "{{ item.container_name }}"
name: node
image: newsblur/newsblur_node
state: started
command: "{{ item.command }}"
command: node /srv/node/newsblur.js
container_default_behavior: no_defaults
pull: true
network_mode: host
@ -15,23 +15,11 @@
volumes:
- /srv/newsblur/node:/srv/node
with_items:
- container_name: node_socket
ports: 8888:8888
command: node /srv/node/unread_counts.js
target_host: node-socket
- container_name: original_pages
ports: 3060:3060
command: node /srv/node/original_page.js
target_host: node-page
- container_name: original_text
ports: 4040:4040
command: node /srv/node/original_text.js
target_host: node-text
- container_name: favicons
ports: 3030:3030
command: node /srv/node/favicons.js
target_host: node-favicons
when: item.target_host in inventory_hostname
- node-socket
- node-page
- node-text
- node-favicons
when: item in inventory_hostname
- name: Start non-newsblur node docker containers
become: yes
@ -59,18 +47,13 @@
become: yes
template:
src: consul_service.json
dest: /etc/consul.d/{{item.target_host}}.json
dest: /etc/consul.d/{{item}}.json
with_items:
- port: 8888
target_host: node-socket
- port: 3060
target_host: node-page
- port: 4040
target_host: node-text
- port: 3030
target_host: node-favicons
- port: 8080
target_host: node-images
- node-socket
- node-page
- node-text
- node-favicons
- node-images
notify:
- reload consul
when: item.target_host in inventory_hostname and disable_consul_services is not defined
when: item in inventory_hostname and disable_consul_services is not defined

15
ansible/roles/node/templates/consul_service.json
View file

@ -1,11 +1,12 @@
{
"service": {
"name": "{{ item.target_host }}",
"id": "{{ inventory_hostname }}",
"name": "{{ inventory_hostname|regex_replace('\d+', '') }}",
"id": "{{ inventory_hostname }}-web",
"tags": [
"node"
"web"
],
"port": {{ item.port }}
"port": 8008
{% if item.target_host in ["node-images", "node-favicons", "node-text", "node-page"] %}
,
"checks": [{
@ -13,11 +14,11 @@
{% if item.target_host == "node-images" %}
"http": "http://{{ ansible_ssh_host }}:8080/sc,sN1megONJiGNy-CCvqzVPTv-TWRhgSKhFlf61XAYESl4=/http:/samuelclay.com/static/images/2019%20-%20Cuba.jpg",
{% elif item.target_host == "node-favicons" %}
"http": "http://{{ ansible_ssh_host }}:3030/rss_feeds/icon/1",
"http": "http://{{ ansible_ssh_host }}:8008/rss_feeds/icon/1",
{% elif item.target_host == "node-text" %}
"http": "http://{{ ansible_ssh_host }}:4040/rss_feeds/original_text_fetcher?test=1",
"http": "http://{{ ansible_ssh_host }}:8008/rss_feeds/original_text_fetcher?test=1",
{% elif item.target_host == "node-page" %}
"http": "http://{{ ansible_ssh_host }}:3060/original_page/1?test=1",
"http": "http://{{ ansible_ssh_host }}:8008/original_page/1?test=1",
{% endif %}
"interval": "15s"
}]

2
apps/rss_feeds/models.py
View file

@ -1688,10 +1688,10 @@ class Feed(models.Model):
return sorted_popularity
def well_read_score(self):
"""Average percentage of stories read vs published across recently active subscribers"""
from apps.reader.models import UserSubscription
from apps.social.models import MSharedStory
# Average percentage of stories read vs published across recently active subscribers
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
p = r.pipeline()

44
docker-compose.yml
View file

@ -92,51 +92,18 @@ services:
volumes:
- ./docker/volumes/db_mongo:/data/db
original_pages:
newsblur_node:
image: newsblur/newsblur_node
container_name: node_original_page
container_name: node
environment:
- NODE_ENV=development
command: node original_page.js
ports:
- 3060:3060
volumes:
- node-files:/srv
original_text:
image: newsblur/newsblur_node
container_name: node_original_text
environment:
- NODE_ENV=development
ports:
- 4040:4040
command: node original_text.js
volumes:
- node-files:/srv
favicons:
image: newsblur/newsblur_node
container_name: node_favicons
environment:
- NODE_ENV=docker
- MONGODB_PORT=29019
ports:
- 3030:3030
command: node favicons.js
command: node newsblur.js
stop_signal: HUP
depends_on:
- db_postgres
volumes:
- node-files:/srv
node_websocket:
container_name: node_websocket
image: newsblur/newsblur_node
environment:
- NODE_ENV=docker
command: node unread_counts.js
stop_signal: HUP
ports:
- 8888:8888
- 8008:8008
volumes:
- node-files:/srv
@ -167,6 +134,7 @@ services:
depends_on:
- nginx
- newsblur_web
- newsblur_node
- db_redis
- db_postgres
- db_elasticsearch

8
docker/haproxy/haproxy.consul.cfg.j2
View file

@ -114,26 +114,26 @@ backend node_socket
balance roundrobin
default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
{% for host in groups.node_socket %}
server {{host}} {{host}}.node.nyc1.consul:8888
server {{host}} {{host}}.node.nyc1.consul:8008
{% endfor %}
backend node_favicon
http-check expect rstatus 200|503
option httpchk GET /rss_feeds/icon/1
balance roundrobin
server-template node-favicons 1 _node-favicons._tcp.service.nyc1.consul:3030 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
server-template node-favicons 1 _node-favicons._tcp.service.nyc1.consul:8008 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
backend node_text
http-check expect rstatus 200|503
option httpchk GET /rss_feeds/original_text_fetcher?test=1
balance roundrobin
server-template node-text 1 _node-text._tcp.service.nyc1.consul:4040 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
server-template node-text 1 _node-text._tcp.service.nyc1.consul:8008 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
backend node_page
http-check expect rstatus 200|503
option httpchk GET /original_page/1?test=1
balance roundrobin
server-template node-page 1 _node-page._tcp.service.nyc1.consul:3060 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
server-template node-page 1 _node-page._tcp.service.nyc1.consul:8008 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
backend postgres
option httpchk GET /db_check/postgres

12
docker/haproxy/haproxy.docker-compose.cfg
View file

@ -51,6 +51,7 @@ frontend public
use_backend node_socket if { path_beg /v3/socket.io/ }
use_backend node_favicon if { path_beg /rss_feeds/icon/ }
use_backend node_text if { path_beg /rss_feeds/original_text_fetcher }
use_backend node_page if { path_beg /original_page/ }
use_backend nginx if { path_beg /media/ }
use_backend nginx if { path_beg /static/ }
use_backend nginx if { path_beg /favicon }
@ -65,20 +66,25 @@ frontend public
backend node_socket
http-check expect rstatus 200|503
balance roundrobin
server node_socket node_websocket:8888 check inter 3000ms
server node_socket newsblur_node:8008 check inter 3000ms
backend node_favicon
http-check expect rstatus 200|301|503
# option httpchk GET /rss_feeds/icon/1
balance roundrobin
server node_favicon favicons:3030 check inter 3000ms
server node_favicon newsblur_node:8008 check inter 3000ms
backend node_text
http-check expect rstatus 200|503
option httpchk GET /rss_feeds/original_text_fetcher?test=1
balance roundrobin
server node_text original_text:4040 check inter 3000ms
server node_text newsblur_node:8008 check inter 3000ms
backend node_page
http-check expect rstatus 200|503
option httpchk GET /rss_feeds/original_text_fetcher?test=1
balance roundrobin
server node_page newsblur_node:8008 check inter 3000ms
backend nginx
balance roundrobin

116
node/favicons.coffee
View file

@ -1,69 +1,67 @@
app = require('express')()
server = require('http').Server(app)
mongo = require 'mongodb'
log = require './log.js'
favicons = (app) =>
ENV_DEV = process.env.NODE_ENV == 'development'
ENV_PROD = process.env.NODE_ENV == 'production'
ENV_DOCKER = process.env.NODE_ENV == 'docker'
MONGODB_SERVER = "db_mongo"
if ENV_DEV
MONGODB_SERVER = 'localhost'
else if ENV_PROD
MONGODB_SERVER = 'db-mongo.service.nyc1.consul'
MONGODB_PORT = parseInt(process.env.MONGODB_PORT or 27017, 10)
ENV_DEV = process.env.NODE_ENV == 'development'
ENV_PROD = process.env.NODE_ENV == 'production'
ENV_DOCKER = process.env.NODE_ENV == 'docker'
MONGODB_SERVER = "db_mongo"
if ENV_DEV
MONGODB_SERVER = 'localhost'
else if ENV_PROD
MONGODB_SERVER = 'db-mongo.service.nyc1.consul'
MONGODB_PORT = parseInt(process.env.MONGODB_PORT or 27017, 10)
log.debug "Starting NewsBlur Favicon server..."
if !ENV_DEV and !process.env.NODE_ENV
log.debug "Specify NODE_ENV=<development,docker,production>"
return
else if ENV_DEV
log.debug "Running as development server"
else if ENV_DOCKER
log.debug "Running as docker server"
else
log.debug "Running as production server"
if ENV_DEV or ENV_DOCKER
url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur"
else
url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred"
do ->
try
client = mongo.MongoClient url, useUnifiedTopology: true
await client.connect()
catch err
log.debug "Error connecting to Mongo: #{err}"
log.debug "Starting NewsBlur Favicon server..."
if !ENV_DEV and !process.env.NODE_ENV
log.debug "Specify NODE_ENV=<development,docker,production>"
return
else if ENV_DEV
log.debug "Running as development server"
else if ENV_DOCKER
log.debug "Running as docker server"
else
log.debug "Running as production server"
if ENV_DEV or ENV_DOCKER
url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur"
else
url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred"
db = client.db "newsblur"
collection = db.collection "feed_icons"
do ->
try
client = mongo.MongoClient url, useUnifiedTopology: true
await client.connect()
catch err
log.debug "Error connecting to Mongo: #{err}"
return
log.debug "Connected to #{db?.serverConfig.s.seedlist[0].host}:#{db?.serverConfig.s.seedlist[0].port}"
if err
log.debug " ***> Error connecting: #{err}"
db = client.db "newsblur"
collection = db.collection "feed_icons"
app.get /\/rss_feeds\/icon\/(\d+)\/?/, (req, res) =>
feed_id = parseInt(req.params[0], 10)
etag = req.header('If-None-Match')
log.debug "Feed: #{feed_id} " + if etag then " / #{etag}" else ""
collection.findOne _id: feed_id, (err, docs) ->
if not err and etag and docs and docs?.color == etag
log.debug "Cached: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else ""
res.sendStatus 304
else if not err and docs and docs.data
log.debug "Req: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else ""
res.header 'etag', docs.color
body = new Buffer(docs.data, 'base64')
res.set("Content-Type", "image/png")
res.status(200).send body
else
log.debug "Redirect: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else ""
if ENV_DEV
res.redirect '/media/img/icons/circular/world.png'
log.debug "Connected to #{db?.serverConfig.s.seedlist[0].host}:#{db?.serverConfig.s.seedlist[0].port}"
if err
log.debug " ***> Error connecting: #{err}"
app.get /\/rss_feeds\/icon\/(\d+)\/?/, (req, res) =>
feed_id = parseInt(req.params[0], 10)
etag = req.header('If-None-Match')
log.debug "Feed: #{feed_id} " + if etag then " / #{etag}" else ""
collection.findOne _id: feed_id, (err, docs) ->
if not err and etag and docs and docs?.color == etag
log.debug "Cached: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else ""
res.sendStatus 304
else if not err and docs and docs.data
log.debug "Req: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else ""
res.header 'etag', docs.color
body = new Buffer(docs.data, 'base64')
res.set("Content-Type", "image/png")
res.status(200).send body
else
res.redirect 'https://www.newsblur.com/media/img/icons/circular/world.png'
log.debug "Redirect: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else ""
if ENV_DEV
res.redirect '/media/img/icons/circular/world.png'
else
res.redirect 'https://www.newsblur.com/media/img/icons/circular/world.png'
app.listen 3030
exports.favicons = favicons

155
node/favicons.js
View file

@ -1,97 +1,88 @@
// Generated by CoffeeScript 2.5.1
(function() {
var ENV_DEV, ENV_DOCKER, ENV_PROD, MONGODB_PORT, MONGODB_SERVER, app, log, mongo, server, url;
app = require('express')();
server = require('http').Server(app);
var favicons, log, mongo;
mongo = require('mongodb');
log = require('./log.js');
ENV_DEV = process.env.NODE_ENV === 'development';
ENV_PROD = process.env.NODE_ENV === 'production';
ENV_DOCKER = process.env.NODE_ENV === 'docker';
MONGODB_SERVER = "db_mongo";
if (ENV_DEV) {
MONGODB_SERVER = 'localhost';
} else if (ENV_PROD) {
MONGODB_SERVER = 'db-mongo.service.nyc1.consul';
}
MONGODB_PORT = parseInt(process.env.MONGODB_PORT || 27017, 10);
log.debug("Starting NewsBlur Favicon server...");
if (!ENV_DEV && !process.env.NODE_ENV) {
log.debug("Specify NODE_ENV=<development,docker,production>");
return;
} else if (ENV_DEV) {
log.debug("Running as development server");
} else if (ENV_DOCKER) {
log.debug("Running as docker server");
} else {
log.debug("Running as production server");
}
if (ENV_DEV || ENV_DOCKER) {
url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur`;
} else {
url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred`;
}
(async function() {
var client, collection, db, err;
try {
client = mongo.MongoClient(url, {
useUnifiedTopology: true
});
await client.connect();
} catch (error) {
err = error;
log.debug(`Error connecting to Mongo: ${err}`);
favicons = (app) => {
var ENV_DEV, ENV_DOCKER, ENV_PROD, MONGODB_PORT, MONGODB_SERVER, url;
ENV_DEV = process.env.NODE_ENV === 'development';
ENV_PROD = process.env.NODE_ENV === 'production';
ENV_DOCKER = process.env.NODE_ENV === 'docker';
MONGODB_SERVER = "db_mongo";
if (ENV_DEV) {
MONGODB_SERVER = 'localhost';
} else if (ENV_PROD) {
MONGODB_SERVER = 'db-mongo.service.nyc1.consul';
}
MONGODB_PORT = parseInt(process.env.MONGODB_PORT || 27017, 10);
log.debug("Starting NewsBlur Favicon server...");
if (!ENV_DEV && !process.env.NODE_ENV) {
log.debug("Specify NODE_ENV=<development,docker,production>");
return;
} else if (ENV_DEV) {
log.debug("Running as development server");
} else if (ENV_DOCKER) {
log.debug("Running as docker server");
} else {
log.debug("Running as production server");
}
db = client.db("newsblur");
collection = db.collection("feed_icons");
log.debug(`Connected to ${db != null ? db.serverConfig.s.seedlist[0].host : void 0}:${db != null ? db.serverConfig.s.seedlist[0].port : void 0}`);
if (err) {
log.debug(` ***> Error connecting: ${err}`);
if (ENV_DEV || ENV_DOCKER) {
url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur`;
} else {
url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred`;
}
app.get(/\/rss_feeds\/icon\/(\d+)\/?/, (req, res) => {
var etag, feed_id;
feed_id = parseInt(req.params[0], 10);
etag = req.header('If-None-Match');
log.debug(`Feed: ${feed_id} ` + (etag ? ` / ${etag}` : ""));
return collection.findOne({
_id: feed_id
}, function(err, docs) {
var body;
if (!err && etag && docs && (docs != null ? docs.color : void 0) === etag) {
log.debug(`Cached: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : ""));
return res.sendStatus(304);
} else if (!err && docs && docs.data) {
log.debug(`Req: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : ""));
res.header('etag', docs.color);
body = new Buffer(docs.data, 'base64');
res.set("Content-Type", "image/png");
return res.status(200).send(body);
} else {
log.debug(`Redirect: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : ""));
if (ENV_DEV) {
return res.redirect('/media/img/icons/circular/world.png');
return (async function() {
var client, collection, db, err;
try {
client = mongo.MongoClient(url, {
useUnifiedTopology: true
});
await client.connect();
} catch (error) {
err = error;
log.debug(`Error connecting to Mongo: ${err}`);
return;
}
db = client.db("newsblur");
collection = db.collection("feed_icons");
log.debug(`Connected to ${db != null ? db.serverConfig.s.seedlist[0].host : void 0}:${db != null ? db.serverConfig.s.seedlist[0].port : void 0}`);
if (err) {
log.debug(` ***> Error connecting: ${err}`);
}
return app.get(/\/rss_feeds\/icon\/(\d+)\/?/, (req, res) => {
var etag, feed_id;
feed_id = parseInt(req.params[0], 10);
etag = req.header('If-None-Match');
log.debug(`Feed: ${feed_id} ` + (etag ? ` / ${etag}` : ""));
return collection.findOne({
_id: feed_id
}, function(err, docs) {
var body;
if (!err && etag && docs && (docs != null ? docs.color : void 0) === etag) {
log.debug(`Cached: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : ""));
return res.sendStatus(304);
} else if (!err && docs && docs.data) {
log.debug(`Req: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : ""));
res.header('etag', docs.color);
body = new Buffer(docs.data, 'base64');
res.set("Content-Type", "image/png");
return res.status(200).send(body);
} else {
return res.redirect('https://www.newsblur.com/media/img/icons/circular/world.png');
log.debug(`Redirect: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : ""));
if (ENV_DEV) {
return res.redirect('/media/img/icons/circular/world.png');
} else {
return res.redirect('https://www.newsblur.com/media/img/icons/circular/world.png');
}
}
}
});
});
});
return app.listen(3030);
})();
})();
};
exports.favicons = favicons;
}).call(this);

16
node/newsblur.coffee Normal file
View file

@ -0,0 +1,16 @@
app = require('express')()
server = require('http').Server(app)
log = require './log.js'
original_page = require('./original_page.js').original_page
original_text = require('./original_text.js').original_text
favicons = require('./favicons.js').favicons
unread_counts = require('./unread_counts.js').unread_counts
original_page(app)
original_text(app)
favicons(app)
unread_counts(server)
log.debug "Starting NewsBlur Node Server"
app.listen(8008)

31
node/newsblur.js Normal file
View file

@ -0,0 +1,31 @@
// Generated by CoffeeScript 2.5.1
(function() {
var app, favicons, log, original_page, original_text, server, unread_counts;
app = require('express')();
server = require('http').Server(app);
log = require('./log.js');
original_page = require('./original_page.js').original_page;
original_text = require('./original_text.js').original_text;
favicons = require('./favicons.js').favicons;
unread_counts = require('./unread_counts.js').unread_counts;
original_page(app);
original_text(app);
favicons(app);
unread_counts(server);
log.debug("Starting NewsBlur Node Server");
app.listen(8008);
}).call(this);

113
node/original_page.coffee
View file

@ -1,69 +1,70 @@
app = require('express')()
server = require('http').Server(app)
path = require 'path'
busboy = require 'connect-busboy'
fs = require 'fs'
mkdirp = require 'mkdirp'
log = require './log.js'
DEV = process.env.NODE_ENV == 'development'
original_page = (app) =>
DEV = process.env.NODE_ENV == 'development'
DB_PATH = if DEV then 'originals' else '/srv/originals'
DB_PATH = if DEV then 'originals' else '/srv/originals'
server.listen 3060
app.use busboy()
app.use busboy()
app.get /^\/original_page\/(\d+)\/?/, (req, res) =>
if req.query.test
return res.end "OK"
app.get /^\/original_page\/(\d+)\/?/, (req, res) =>
if req.query.test
return res.end "OK"
feedId = parseInt(req.params[0], 10)
etag = req.header('If-None-Match')
lastModified = req.header('If-Modified-Since')
feedIdDir = splitFeedId feedId
filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
fs.exists filePath, (exists, err) ->
console.log " ---> Loading: #{feedId} (#{filePath}). " +
"#{if exists then "" else "NOT FOUND"}"
if not exists
return res.send 404
fs.stat filePath, (err, stats) ->
if not err and etag and stats.mtime == etag
return res.send 304
if not err and lastModified and stats.mtime == lastModified
return res.send 304
fs.readFile filePath, (err, content) ->
res.header 'Etag', Date.parse(stats.mtime)
res.send content
app.post /^\/original_page\/(\d+)\/?/, (req, res) =>
feedId = parseInt(req.params[0], 10)
feedIdDir = splitFeedId feedId
req.pipe req.busboy
req.busboy.on 'file', (fieldname, file, filename) ->
# console.log "Uploading #{fieldname} / #{file} / #{filename}"
feedId = parseInt(req.params[0], 10)
etag = req.header('If-None-Match')
lastModified = req.header('If-Modified-Since')
feedIdDir = splitFeedId feedId
filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
filePathDir = path.dirname filePath
mkdirp filePathDir, (err) ->
console.log err if err
fstream = fs.createWriteStream filePath
file.pipe fstream
fstream.on 'close', ->
fs.stat filePath, (err, stats) ->
console.log err if err
console.log " ---> Saving: #{feedId} (#{filePath}) #{stats.size} bytes"
res.send "OK"
fs.exists filePath, (exists, err) ->
log.debug "Loading: #{feedId} (#{filePath}). " +
"#{if exists then "" else "NOT FOUND"}"
if not exists
return res.send 404
fs.stat filePath, (err, stats) ->
if not err and etag and stats.mtime == etag
return res.send 304
if not err and lastModified and stats.mtime == lastModified
return res.send 304
fs.readFile filePath, (err, content) ->
res.header 'Etag', Date.parse(stats.mtime)
res.send content
splitFeedId = (feedId) ->
feedId += ''
# x2 = if feedId.length > 1 then '.' + feedId[1] else ''
rgx = /(\d+)(\d{3})/
feedId = feedId.replace rgx, '$1' + '/' + '$2' while rgx.test(feedId)
return feedId;
app.post /^\/original_page\/(\d+)\/?/, (req, res) =>
feedId = parseInt(req.params[0], 10)
feedIdDir = splitFeedId feedId
req.pipe req.busboy
req.busboy.on 'file', (fieldname, file, filename) ->
# log.debug "Uploading #{fieldname} / #{file} / #{filename}"
filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
filePathDir = path.dirname filePath
mkdirp filePathDir, (err) ->
log.debug err if err
fstream = fs.createWriteStream filePath
file.pipe fstream
fstream.on 'close', ->
fs.stat filePath, (err, stats) ->
log.debug err if err
log.debug "Saving: #{feedId} (#{filePath}) #{stats.size} bytes"
res.send "OK"
console.log " ---> Starting Original Page server #{if DEV then "on DEV" else "in production"}"
splitFeedId = (feedId) ->
feedId += ''
# x2 = if feedId.length > 1 then '.' + feedId[1] else ''
rgx = /(\d+)(\d{3})/
feedId = feedId.replace rgx, '$1' + '/' + '$2' while rgx.test(feedId)
return feedId;
log.debug "Starting Original Page server #{if DEV then "on DEV" else "in production"}"
exports.original_page = original_page

145
node/original_page.js
View file

@ -1,10 +1,6 @@
// Generated by CoffeeScript 2.5.1
(function() {
var DB_PATH, DEV, app, busboy, fs, mkdirp, path, server, splitFeedId;
app = require('express')();
server = require('http').Server(app);
var busboy, fs, log, mkdirp, original_page, path;
path = require('path');
@ -14,85 +10,84 @@
mkdirp = require('mkdirp');
DEV = process.env.NODE_ENV === 'development';
log = require('./log.js');
DB_PATH = DEV ? 'originals' : '/srv/originals';
server.listen(3060);
app.use(busboy());
app.get(/^\/original_page\/(\d+)\/?/, (req, res) => {
var etag, feedId, feedIdDir, filePath, lastModified;
if (req.query.test) {
return res.end("OK");
}
feedId = parseInt(req.params[0], 10);
etag = req.header('If-None-Match');
lastModified = req.header('If-Modified-Since');
feedIdDir = splitFeedId(feedId);
filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
return fs.exists(filePath, function(exists, err) {
console.log(` ---> Loading: ${feedId} (${filePath}). ` + `${exists ? "" : "NOT FOUND"}`);
if (!exists) {
return res.send(404);
original_page = (app) => {
var DB_PATH, DEV, splitFeedId;
DEV = process.env.NODE_ENV === 'development';
DB_PATH = DEV ? 'originals' : '/srv/originals';
app.use(busboy());
app.get(/^\/original_page\/(\d+)\/?/, (req, res) => {
var etag, feedId, feedIdDir, filePath, lastModified;
if (req.query.test) {
return res.end("OK");
}
return fs.stat(filePath, function(err, stats) {
if (!err && etag && stats.mtime === etag) {
return res.send(304);
}
if (!err && lastModified && stats.mtime === lastModified) {
return res.send(304);
}
return fs.readFile(filePath, function(err, content) {
res.header('Etag', Date.parse(stats.mtime));
return res.send(content);
});
});
});
});
app.post(/^\/original_page\/(\d+)\/?/, (req, res) => {
var feedId, feedIdDir;
feedId = parseInt(req.params[0], 10);
feedIdDir = splitFeedId(feedId);
req.pipe(req.busboy);
return req.busboy.on('file', function(fieldname, file, filename) {
var filePath, filePathDir;
// console.log "Uploading #{fieldname} / #{file} / #{filename}"
feedId = parseInt(req.params[0], 10);
etag = req.header('If-None-Match');
lastModified = req.header('If-Modified-Since');
feedIdDir = splitFeedId(feedId);
filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
filePathDir = path.dirname(filePath);
return mkdirp(filePathDir, function(err) {
var fstream;
if (err) {
console.log(err);
return fs.exists(filePath, function(exists, err) {
log.debug(`Loading: ${feedId} (${filePath}). ` + `${exists ? "" : "NOT FOUND"}`);
if (!exists) {
return res.send(404);
}
fstream = fs.createWriteStream(filePath);
file.pipe(fstream);
return fstream.on('close', function() {
return fs.stat(filePath, function(err, stats) {
if (err) {
console.log(err);
}
console.log(` ---> Saving: ${feedId} (${filePath}) ${stats.size} bytes`);
return res.send("OK");
return fs.stat(filePath, function(err, stats) {
if (!err && etag && stats.mtime === etag) {
return res.send(304);
}
if (!err && lastModified && stats.mtime === lastModified) {
return res.send(304);
}
return fs.readFile(filePath, function(err, content) {
res.header('Etag', Date.parse(stats.mtime));
return res.send(content);
});
});
});
});
});
splitFeedId = function(feedId) {
var rgx;
feedId += '';
// x2 = if feedId.length > 1 then '.' + feedId[1] else ''
rgx = /(\d+)(\d{3})/;
while (rgx.test(feedId)) {
feedId = feedId.replace(rgx, '$1' + '/' + '$2');
}
return feedId;
app.post(/^\/original_page\/(\d+)\/?/, (req, res) => {
var feedId, feedIdDir;
feedId = parseInt(req.params[0], 10);
feedIdDir = splitFeedId(feedId);
req.pipe(req.busboy);
return req.busboy.on('file', function(fieldname, file, filename) {
var filePath, filePathDir;
// log.debug "Uploading #{fieldname} / #{file} / #{filename}"
filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
filePathDir = path.dirname(filePath);
return mkdirp(filePathDir, function(err) {
var fstream;
if (err) {
log.debug(err);
}
fstream = fs.createWriteStream(filePath);
file.pipe(fstream);
return fstream.on('close', function() {
return fs.stat(filePath, function(err, stats) {
if (err) {
log.debug(err);
}
log.debug(`Saving: ${feedId} (${filePath}) ${stats.size} bytes`);
return res.send("OK");
});
});
});
});
});
splitFeedId = function(feedId) {
var rgx;
feedId += '';
// x2 = if feedId.length > 1 then '.' + feedId[1] else ''
rgx = /(\d+)(\d{3})/;
while (rgx.test(feedId)) {
feedId = feedId.replace(rgx, '$1' + '/' + '$2');
}
return feedId;
};
return log.debug(`Starting Original Page server ${DEV ? "on DEV" : "in production"}`);
};
console.log(` ---> Starting Original Page server ${DEV ? "on DEV" : "in production"}`);
exports.original_page = original_page;
}).call(this);

57
node/original_text.coffee
View file

@ -1,36 +1,35 @@
Mercury = require('@postlight/mercury-parser')
app = require('express')()
server = require('http').Server(app)
log = require './log.js'
DEV = process.env.NODE_ENV == 'development'
original_text = (app) =>
DEV = process.env.NODE_ENV == 'development'
log.debug "Starting NewsBlur Original Text Fetcher / Mercury Parser..."
if !DEV and !process.env.NODE_ENV
log.debug "Specify NODE_ENV=<development,production>"
return
else if DEV
log.debug "Running as development server"
else
log.debug "Running as production server"
log.debug "Starting NewsBlur Original Text Fetcher / Mercury Parser..."
if !DEV and !process.env.NODE_ENV
log.debug "Specify NODE_ENV=<development,production>"
return
else if DEV
log.debug "Running as development server"
else
log.debug "Running as production server"
app.get /\/rss_feeds\/original_text_fetcher\/?/, (req, res) =>
res.setHeader 'Content-Type', 'application/json'
if req.query.test
return res.end "OK"
url = req.query.url
if !url
log.debug "Missing url"
return res.end JSON.stringify error: "Missing `url` query parameter."
api_key = req.header('x-api-key') or req.query.apikey
if not DEV and (!api_key or api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") == -1)
log.debug "Mismatched API key: #{url} / #{api_key}"
return res.end JSON.stringify error: "Invalid API key. You need to set up your own Original Text server."
app.get /\/rss_feeds\/original_text_fetcher\/?/, (req, res) =>
res.setHeader 'Content-Type', 'application/json'
if req.query.test
return res.end "OK"
Mercury.parse(url).then (result) =>
log.debug "Fetched: #{url}"
res.end JSON.stringify result
url = req.query.url
if !url
log.debug "Missing url"
return res.end JSON.stringify error: "Missing `url` query parameter."
api_key = req.header('x-api-key') or req.query.apikey
if not DEV and (!api_key or api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") == -1)
log.debug "Mismatched API key: #{url} / #{api_key}"
return res.end JSON.stringify error: "Invalid API key. You need to set up your own Original Text server."
Mercury.parse(url).then (result) =>
log.debug "Fetched: #{url}"
res.end JSON.stringify result
app.listen 4040
exports.original_text = original_text

80
node/original_text.js
View file

@ -1,54 +1,50 @@
// Generated by CoffeeScript 2.5.1
(function() {
var DEV, Mercury, app, log, server;
var Mercury, log, original_text;
Mercury = require('@postlight/mercury-parser');
app = require('express')();
server = require('http').Server(app);
log = require('./log.js');
DEV = process.env.NODE_ENV === 'development';
log.debug("Starting NewsBlur Original Text Fetcher / Mercury Parser...");
if (!DEV && !process.env.NODE_ENV) {
log.debug("Specify NODE_ENV=<development,production>");
return;
} else if (DEV) {
log.debug("Running as development server");
} else {
log.debug("Running as production server");
}
app.get(/\/rss_feeds\/original_text_fetcher\/?/, (req, res) => {
var api_key, url;
res.setHeader('Content-Type', 'application/json');
if (req.query.test) {
return res.end("OK");
original_text = (app) => {
var DEV;
DEV = process.env.NODE_ENV === 'development';
log.debug("Starting NewsBlur Original Text Fetcher / Mercury Parser...");
if (!DEV && !process.env.NODE_ENV) {
log.debug("Specify NODE_ENV=<development,production>");
return;
} else if (DEV) {
log.debug("Running as development server");
} else {
log.debug("Running as production server");
}
url = req.query.url;
if (!url) {
log.debug("Missing url");
return res.end(JSON.stringify({
error: "Missing `url` query parameter."
}));
}
api_key = req.header('x-api-key') || req.query.apikey;
if (!DEV && (!api_key || api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") === -1)) {
log.debug(`Mismatched API key: ${url} / ${api_key}`);
return res.end(JSON.stringify({
error: "Invalid API key. You need to set up your own Original Text server."
}));
}
return Mercury.parse(url).then((result) => {
log.debug(`Fetched: ${url}`);
return res.end(JSON.stringify(result));
return app.get(/\/rss_feeds\/original_text_fetcher\/?/, (req, res) => {
var api_key, url;
res.setHeader('Content-Type', 'application/json');
if (req.query.test) {
return res.end("OK");
}
url = req.query.url;
if (!url) {
log.debug("Missing url");
return res.end(JSON.stringify({
error: "Missing `url` query parameter."
}));
}
api_key = req.header('x-api-key') || req.query.apikey;
if (!DEV && (!api_key || api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") === -1)) {
log.debug(`Mismatched API key: ${url} / ${api_key}`);
return res.end(JSON.stringify({
error: "Invalid API key. You need to set up your own Original Text server."
}));
}
return Mercury.parse(url).then((result) => {
log.debug(`Fetched: ${url}`);
return res.end(JSON.stringify(result));
});
});
});
};
app.listen(4040);
exports.original_text = original_text;
}).call(this);

166
node/unread_counts.coffee
View file

@ -2,103 +2,87 @@ fs = require 'fs'
redis = require 'redis'
log = require './log.js'
ENV_DEV = process.env.NODE_ENV == 'development'
ENV_PROD = process.env.NODE_ENV == 'production'
ENV_DOCKER = process.env.NODE_ENV == 'docker'
REDIS_SERVER = "redis"
if ENV_DEV
REDIS_SERVER = 'localhost'
else if ENV_PROD
REDIS_SERVER = 'db-redis-user.service.nyc1.consul'
SECURE = !!process.env.NODE_SSL
REDIS_PORT = if ENV_DOCKER then 6579 else 6379
unread_counts = (server) =>
ENV_DEV = process.env.NODE_ENV == 'development'
ENV_PROD = process.env.NODE_ENV == 'production'
ENV_DOCKER = process.env.NODE_ENV == 'docker'
REDIS_SERVER = "redis"
if ENV_DEV
REDIS_SERVER = 'localhost'
else if ENV_PROD
REDIS_SERVER = 'db-redis-user.service.nyc1.consul'
SECURE = !!process.env.NODE_SSL
REDIS_PORT = if ENV_DOCKER then 6579 else 6379
# client = redis.createClient 6379, REDIS_SERVER
# client = redis.createClient 6379, REDIS_SERVER
# RedisStore = require 'socket.io/lib/stores/redis'
# rpub = redis.createClient 6379, REDIS_SERVER
# rsub = redis.createClient 6379, REDIS_SERVER
# rclient = redis.createClient 6379, REDIS_SERVER
# RedisStore = require 'socket.io/lib/stores/redis'
# rpub = redis.createClient 6379, REDIS_SERVER
# rsub = redis.createClient 6379, REDIS_SERVER
# rclient = redis.createClient 6379, REDIS_SERVER
log.debug "Starting NewsBlur unread count server..."
if !ENV_DEV and !process.env.NODE_ENV
log.debug "Specify NODE_ENV=<development,production>"
return
else if ENV_DEV
log.debug "Running as development server"
else
log.debug "Running as production server"
if SECURE
privateKey = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.key').toString()
certificate = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.crt').toString()
# ca = fs.readFileSync('./config/certificates/intermediate.crt').toString()
options =
port: 8889
key: privateKey
cert: certificate
app = require('https').createServer options
io = require('socket.io')(app, path: "/v3/socket.io")
app.listen options.port
log.debug "Listening securely on port #{options.port}"
else
options =
port: 8888
app = require('http').createServer()
io = require('socket.io')(app, path: "/v3/socket.io")
app.listen options.port
log.debug "Listening on port #{options.port}"
# io.set('transports', ['websocket'])
# io.set 'store', new RedisStore
# redisPub : rpub
# redisSub : rsub
# redisClient : rclient
io.on 'connection', (socket) ->
ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address
socket.on 'subscribe:feeds', (@feeds, @username) =>
log.info @username, "Connecting (#{@feeds.length} feeds, #{ip})," +
" (#{io.engine.clientsCount} connected) " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}"
log.debug "Starting NewsBlur unread count server..."
if !ENV_DEV and !process.env.NODE_ENV
log.debug "Specify NODE_ENV=<development,production>"
return
else if ENV_DEV
log.debug "Running as development server"
else
log.debug "Running as production server"
if not @username
return
socket.on "error", (err) ->
log.debug "Error (socket): #{err}"
socket.subscribe?.quit()
socket.subscribe = redis.createClient REDIS_PORT, REDIS_SERVER
socket.subscribe.on "error", (err) =>
log.info @username, "Error: #{err} (#{@feeds.length} feeds)"
socket.subscribe?.quit()
socket.subscribe.on "connect", =>
log.info @username, "Connected (#{@feeds.length} feeds, #{ip})," +
" (#{io.engine.clientsCount} connected) " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}"
socket.subscribe.subscribe @feeds
feeds_story = @feeds.map (f) -> "#{f}:story"
socket.subscribe.subscribe feeds_story
socket.subscribe.subscribe @username
io = require('socket.io')(server, path: "/v3/socket.io")
socket.subscribe.on 'message', (channel, message) =>
event_name = 'feed:update'
if channel == @username
event_name = 'user:update'
else if channel.indexOf(':story') >= 0
event_name = 'feed:story:new'
log.info @username, "Update on #{channel}: #{event_name} - #{message}"
socket.emit event_name, channel, message
# io.set('transports', ['websocket'])
socket.on 'disconnect', () =>
socket.subscribe?.quit()
log.info @username, "Disconnect (#{@feeds?.length} feeds, #{ip})," +
" there are now #{io.engine.clientsCount} users. " +
# io.set 'store', new RedisStore
# redisPub : rpub
# redisSub : rsub
# redisClient : rclient
io.on 'connection', (socket) ->
ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address
socket.on 'subscribe:feeds', (@feeds, @username) =>
log.info @username, "Connecting (#{@feeds.length} feeds, #{ip})," +
" (#{io.engine.clientsCount} connected) " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}"
if not @username
return
socket.on "error", (err) ->
log.debug "Error (socket): #{err}"
socket.subscribe?.quit()
socket.subscribe = redis.createClient REDIS_PORT, REDIS_SERVER
socket.subscribe.on "error", (err) =>
log.info @username, "Error: #{err} (#{@feeds.length} feeds)"
socket.subscribe?.quit()
socket.subscribe.on "connect", =>
log.info @username, "Connected (#{@feeds.length} feeds, #{ip})," +
" (#{io.engine.clientsCount} connected) " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}"
socket.subscribe.subscribe @feeds
feeds_story = @feeds.map (f) -> "#{f}:story"
socket.subscribe.subscribe feeds_story
socket.subscribe.subscribe @username
io.sockets.on 'error', (err) ->
log.debug "Error (sockets): #{err}"
socket.subscribe.on 'message', (channel, message) =>
event_name = 'feed:update'
if channel == @username
event_name = 'user:update'
else if channel.indexOf(':story') >= 0
event_name = 'feed:story:new'
log.info @username, "Update on #{channel}: #{event_name} - #{message}"
socket.emit event_name, channel, message
socket.on 'disconnect', () =>
socket.subscribe?.quit()
log.info @username, "Disconnect (#{@feeds?.length} feeds, #{ip})," +
" there are now #{io.engine.clientsCount} users. " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}"
io.sockets.on 'error', (err) ->
log.debug "Error (sockets): #{err}"
exports.unread_counts = unread_counts

205
node/unread_counts.js
View file

@ -1,6 +1,6 @@
// Generated by CoffeeScript 2.5.1
(function() {
var ENV_DEV, ENV_DOCKER, ENV_PROD, REDIS_PORT, REDIS_SERVER, SECURE, app, certificate, fs, io, log, options, privateKey, redis;
var fs, log, redis, unread_counts;
fs = require('fs');
@ -8,130 +8,101 @@
log = require('./log.js');
ENV_DEV = process.env.NODE_ENV === 'development';
unread_counts = (server) => {
var ENV_DEV, ENV_DOCKER, ENV_PROD, REDIS_PORT, REDIS_SERVER, SECURE, io;
ENV_DEV = process.env.NODE_ENV === 'development';
ENV_PROD = process.env.NODE_ENV === 'production';
ENV_DOCKER = process.env.NODE_ENV === 'docker';
REDIS_SERVER = "redis";
if (ENV_DEV) {
REDIS_SERVER = 'localhost';
} else if (ENV_PROD) {
REDIS_SERVER = 'db-redis-user.service.nyc1.consul';
}
SECURE = !!process.env.NODE_SSL;
REDIS_PORT = ENV_DOCKER ? 6579 : 6379;
// client = redis.createClient 6379, REDIS_SERVER
ENV_PROD = process.env.NODE_ENV === 'production';
ENV_DOCKER = process.env.NODE_ENV === 'docker';
REDIS_SERVER = "redis";
if (ENV_DEV) {
REDIS_SERVER = 'localhost';
} else if (ENV_PROD) {
REDIS_SERVER = 'db-redis-user.service.nyc1.consul';
}
SECURE = !!process.env.NODE_SSL;
REDIS_PORT = ENV_DOCKER ? 6579 : 6379;
// client = redis.createClient 6379, REDIS_SERVER
// RedisStore = require 'socket.io/lib/stores/redis'
// rpub = redis.createClient 6379, REDIS_SERVER
// rsub = redis.createClient 6379, REDIS_SERVER
// rclient = redis.createClient 6379, REDIS_SERVER
log.debug("Starting NewsBlur unread count server...");
if (!ENV_DEV && !process.env.NODE_ENV) {
log.debug("Specify NODE_ENV=<development,production>");
return;
} else if (ENV_DEV) {
log.debug("Running as development server");
} else {
log.debug("Running as production server");
}
if (SECURE) {
privateKey = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.key').toString();
certificate = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.crt').toString();
// ca = fs.readFileSync('./config/certificates/intermediate.crt').toString()
options = {
port: 8889,
key: privateKey,
cert: certificate
};
app = require('https').createServer(options);
io = require('socket.io')(app, {
// RedisStore = require 'socket.io/lib/stores/redis'
// rpub = redis.createClient 6379, REDIS_SERVER
// rsub = redis.createClient 6379, REDIS_SERVER
// rclient = redis.createClient 6379, REDIS_SERVER
log.debug("Starting NewsBlur unread count server...");
if (!ENV_DEV && !process.env.NODE_ENV) {
log.debug("Specify NODE_ENV=<development,production>");
return;
} else if (ENV_DEV) {
log.debug("Running as development server");
} else {
log.debug("Running as production server");
}
io = require('socket.io')(server, {
path: "/v3/socket.io"
});
app.listen(options.port);
log.debug(`Listening securely on port ${options.port}`);
} else {
options = {
port: 8888
};
app = require('http').createServer();
io = require('socket.io')(app, {
path: "/v3/socket.io"
});
app.listen(options.port);
log.debug(`Listening on port ${options.port}`);
}
// io.set('transports', ['websocket'])
// io.set('transports', ['websocket'])
// io.set 'store', new RedisStore
// redisPub : rpub
// redisSub : rsub
// redisClient : rclient
io.on('connection', function(socket) {
var ip;
ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address;
socket.on('subscribe:feeds', (feeds, username) => {
var ref;
this.feeds = feeds;
this.username = username;
log.info(this.username, `Connecting (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
if (!this.username) {
return;
}
socket.on("error", function(err) {
return log.debug(`Error (socket): ${err}`);
});
if ((ref = socket.subscribe) != null) {
ref.quit();
}
socket.subscribe = redis.createClient(REDIS_PORT, REDIS_SERVER);
socket.subscribe.on("error", (err) => {
var ref1;
log.info(this.username, `Error: ${err} (${this.feeds.length} feeds)`);
return (ref1 = socket.subscribe) != null ? ref1.quit() : void 0;
});
socket.subscribe.on("connect", () => {
var feeds_story;
log.info(this.username, `Connected (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
socket.subscribe.subscribe(this.feeds);
feeds_story = this.feeds.map(function(f) {
return `${f}:story`;
});
socket.subscribe.subscribe(feeds_story);
return socket.subscribe.subscribe(this.username);
});
return socket.subscribe.on('message', (channel, message) => {
var event_name;
event_name = 'feed:update';
if (channel === this.username) {
event_name = 'user:update';
} else if (channel.indexOf(':story') >= 0) {
event_name = 'feed:story:new';
// io.set 'store', new RedisStore
// redisPub : rpub
// redisSub : rsub
// redisClient : rclient
io.on('connection', function(socket) {
var ip;
ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address;
socket.on('subscribe:feeds', (feeds, username) => {
var ref;
this.feeds = feeds;
this.username = username;
log.info(this.username, `Connecting (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
if (!this.username) {
return;
}
log.info(this.username, `Update on ${channel}: ${event_name} - ${message}`);
return socket.emit(event_name, channel, message);
socket.on("error", function(err) {
return log.debug(`Error (socket): ${err}`);
});
if ((ref = socket.subscribe) != null) {
ref.quit();
}
socket.subscribe = redis.createClient(REDIS_PORT, REDIS_SERVER);
socket.subscribe.on("error", (err) => {
var ref1;
log.info(this.username, `Error: ${err} (${this.feeds.length} feeds)`);
return (ref1 = socket.subscribe) != null ? ref1.quit() : void 0;
});
socket.subscribe.on("connect", () => {
var feeds_story;
log.info(this.username, `Connected (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
socket.subscribe.subscribe(this.feeds);
feeds_story = this.feeds.map(function(f) {
return `${f}:story`;
});
socket.subscribe.subscribe(feeds_story);
return socket.subscribe.subscribe(this.username);
});
return socket.subscribe.on('message', (channel, message) => {
var event_name;
event_name = 'feed:update';
if (channel === this.username) {
event_name = 'user:update';
} else if (channel.indexOf(':story') >= 0) {
event_name = 'feed:story:new';
}
log.info(this.username, `Update on ${channel}: ${event_name} - ${message}`);
return socket.emit(event_name, channel, message);
});
});
return socket.on('disconnect', () => {
var ref, ref1;
if ((ref = socket.subscribe) != null) {
ref.quit();
}
return log.info(this.username, `Disconnect (${(ref1 = this.feeds) != null ? ref1.length : void 0} feeds, ${ip}),` + ` there are now ${io.engine.clientsCount} users. ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
});
});
return socket.on('disconnect', () => {
var ref, ref1;
if ((ref = socket.subscribe) != null) {
ref.quit();
}
return log.info(this.username, `Disconnect (${(ref1 = this.feeds) != null ? ref1.length : void 0} feeds, ${ip}),` + ` there are now ${io.engine.clientsCount} users. ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
return io.sockets.on('error', function(err) {
return log.debug(`Error (sockets): ${err}`);
});
});
};
io.sockets.on('error', function(err) {
return log.debug(`Error (sockets): ${err}`);
});
exports.unread_counts = unread_counts;
}).call(this);

6
templates/analyzer/popularity_query.xhtml
View file

@ -12,12 +12,12 @@
<div class="NB-static-form-wrapper">
<div class="NB-delete-form NB-static-form">
<h2>NewsBlur Publisher Keyword Popularity XLSX Creator for YC Founders</h2>
<h2>NewsBlur News Popularity Spreadsheet Creator</h2>
<h5>Search for topics across millions of stories from millions of publishers</h5>
{% if success %}
<h4 style="color: #1C6130; text-align: center">Got it!<br>Email should be sent within the hour.</h4>
<h4 style="color: #1C6130; text-align: center">Got it!<br>Email should be sent within the next few minutes.</h4>
{% else %}
<form action="" method="POST">{% csrf_token %}
<div class="NB-fields">
@ -43,4 +43,4 @@
</div>
</div>
{% endblock %}
{% endblock %}