public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-09-18 21:50:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merging all four node servers into a single one on port 8008. Updating HAProxy on both consul and docker-compose.

Browse source
This commit is contained in:
Samuel Clay 2021-03-16 19:34:11 -04:00
parent 3f0b2ef366
commit 2906e92499
17 changed files with 578 additions and 638 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
ansible/roles/node/tasks/main.yml
View file

@ -2,10 +2,10 @@
- name: Start node docker containers - name: Start node docker containers
become: yes become: yes
docker_container: docker_container:
name: "{{ item.container_name }}" name: node
image: newsblur/newsblur_node image: newsblur/newsblur_node
state: started state: started
command: "{{ item.command }}" command: node /srv/node/newsblur.js
container_default_behavior: no_defaults container_default_behavior: no_defaults
pull: true pull: true
network_mode: host network_mode: host
@ -15,23 +15,11 @@
volumes: volumes:
- /srv/newsblur/node:/srv/node - /srv/newsblur/node:/srv/node
with_items: with_items:
- container_name: node_socket - node-socket
ports: 8888:8888 - node-page
command: node /srv/node/unread_counts.js - node-text
target_host: node-socket - node-favicons
- container_name: original_pages when: item in inventory_hostname
ports: 3060:3060
command: node /srv/node/original_page.js
target_host: node-page
- container_name: original_text
ports: 4040:4040
command: node /srv/node/original_text.js
target_host: node-text
- container_name: favicons
ports: 3030:3030
command: node /srv/node/favicons.js
target_host: node-favicons
when: item.target_host in inventory_hostname
- name: Start non-newsblur node docker containers - name: Start non-newsblur node docker containers
become: yes become: yes
@ -59,18 +47,13 @@
become: yes become: yes
template: template:
src: consul_service.json src: consul_service.json
dest: /etc/consul.d/{{item.target_host}}.json dest: /etc/consul.d/{{item}}.json
with_items: with_items:
- port: 8888 - node-socket
target_host: node-socket - node-page
- port: 3060 - node-text
target_host: node-page - node-favicons
- port: 4040 - node-images
target_host: node-text
- port: 3030
target_host: node-favicons
- port: 8080
target_host: node-images
notify: notify:
- reload consul - reload consul
when: item.target_host in inventory_hostname and disable_consul_services is not defined when: item in inventory_hostname and disable_consul_services is not defined

15
ansible/roles/node/templates/consul_service.json
View file

@ -1,11 +1,12 @@
{ {
"service": { "service": {
"name": "{{ item.target_host }}", "name": "{{ inventory_hostname|regex_replace('\d+', '') }}",
"id": "{{ inventory_hostname }}", "id": "{{ inventory_hostname }}-web",
"tags": [ "tags": [
"node" "web"
], ],
"port": {{ item.port }} "port": 8008
{% if item.target_host in ["node-images", "node-favicons", "node-text", "node-page"] %} {% if item.target_host in ["node-images", "node-favicons", "node-text", "node-page"] %}
, ,
"checks": [{ "checks": [{
@ -13,11 +14,11 @@
{% if item.target_host == "node-images" %} {% if item.target_host == "node-images" %}
"http": "http://{{ ansible_ssh_host }}:8080/sc,sN1megONJiGNy-CCvqzVPTv-TWRhgSKhFlf61XAYESl4=/http:/samuelclay.com/static/images/2019%20-%20Cuba.jpg", "http": "http://{{ ansible_ssh_host }}:8080/sc,sN1megONJiGNy-CCvqzVPTv-TWRhgSKhFlf61XAYESl4=/http:/samuelclay.com/static/images/2019%20-%20Cuba.jpg",
{% elif item.target_host == "node-favicons" %} {% elif item.target_host == "node-favicons" %}
"http": "http://{{ ansible_ssh_host }}:3030/rss_feeds/icon/1", "http": "http://{{ ansible_ssh_host }}:8008/rss_feeds/icon/1",
{% elif item.target_host == "node-text" %} {% elif item.target_host == "node-text" %}
"http": "http://{{ ansible_ssh_host }}:4040/rss_feeds/original_text_fetcher?test=1", "http": "http://{{ ansible_ssh_host }}:8008/rss_feeds/original_text_fetcher?test=1",
{% elif item.target_host == "node-page" %} {% elif item.target_host == "node-page" %}
"http": "http://{{ ansible_ssh_host }}:3060/original_page/1?test=1", "http": "http://{{ ansible_ssh_host }}:8008/original_page/1?test=1",
{% endif %} {% endif %}
"interval": "15s" "interval": "15s"
}] }]

2
apps/rss_feeds/models.py
View file

@ -1688,10 +1688,10 @@ class Feed(models.Model):
return sorted_popularity return sorted_popularity
def well_read_score(self): def well_read_score(self):
"""Average percentage of stories read vs published across recently active subscribers"""
from apps.reader.models import UserSubscription from apps.reader.models import UserSubscription
from apps.social.models import MSharedStory from apps.social.models import MSharedStory
# Average percentage of stories read vs published across recently active subscribers
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL) r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
p = r.pipeline() p = r.pipeline()

44
docker-compose.yml
View file

@ -92,51 +92,18 @@ services:
volumes: volumes:
- ./docker/volumes/db_mongo:/data/db - ./docker/volumes/db_mongo:/data/db
original_pages: newsblur_node:
image: newsblur/newsblur_node image: newsblur/newsblur_node
container_name: node_original_page container_name: node
environment: environment:
- NODE_ENV=development - NODE_ENV=development
command: node original_page.js
ports:
- 3060:3060
volumes:
- node-files:/srv
original_text:
image: newsblur/newsblur_node
container_name: node_original_text
environment:
- NODE_ENV=development
ports:
- 4040:4040
command: node original_text.js
volumes:
- node-files:/srv
favicons:
image: newsblur/newsblur_node
container_name: node_favicons
environment:
- NODE_ENV=docker
- MONGODB_PORT=29019 - MONGODB_PORT=29019
ports: command: node newsblur.js
- 3030:3030 stop_signal: HUP
command: node favicons.js
depends_on: depends_on:
- db_postgres - db_postgres
volumes:
- node-files:/srv
node_websocket:
container_name: node_websocket
image: newsblur/newsblur_node
environment:
- NODE_ENV=docker
command: node unread_counts.js
stop_signal: HUP
ports: ports:
- 8888:8888 - 8008:8008
volumes: volumes:
- node-files:/srv - node-files:/srv
@ -167,6 +134,7 @@ services:
depends_on: depends_on:
- nginx - nginx
- newsblur_web - newsblur_web
- newsblur_node
- db_redis - db_redis
- db_postgres - db_postgres
- db_elasticsearch - db_elasticsearch

8
docker/haproxy/haproxy.consul.cfg.j2
View file

@ -114,26 +114,26 @@ backend node_socket
balance roundrobin balance roundrobin
default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
{% for host in groups.node_socket %} {% for host in groups.node_socket %}
server {{host}} {{host}}.node.nyc1.consul:8888 server {{host}} {{host}}.node.nyc1.consul:8008
{% endfor %} {% endfor %}
backend node_favicon backend node_favicon
http-check expect rstatus 200|503 http-check expect rstatus 200|503
option httpchk GET /rss_feeds/icon/1 option httpchk GET /rss_feeds/icon/1
balance roundrobin balance roundrobin
server-template node-favicons 1 _node-favicons._tcp.service.nyc1.consul:3030 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none server-template node-favicons 1 _node-favicons._tcp.service.nyc1.consul:8008 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
backend node_text backend node_text
http-check expect rstatus 200|503 http-check expect rstatus 200|503
option httpchk GET /rss_feeds/original_text_fetcher?test=1 option httpchk GET /rss_feeds/original_text_fetcher?test=1
balance roundrobin balance roundrobin
server-template node-text 1 _node-text._tcp.service.nyc1.consul:4040 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none server-template node-text 1 _node-text._tcp.service.nyc1.consul:8008 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
backend node_page backend node_page
http-check expect rstatus 200|503 http-check expect rstatus 200|503
option httpchk GET /original_page/1?test=1 option httpchk GET /original_page/1?test=1
balance roundrobin balance roundrobin
server-template node-page 1 _node-page._tcp.service.nyc1.consul:3060 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none server-template node-page 1 _node-page._tcp.service.nyc1.consul:8008 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
backend postgres backend postgres
option httpchk GET /db_check/postgres option httpchk GET /db_check/postgres

12
docker/haproxy/haproxy.docker-compose.cfg
View file

@ -51,6 +51,7 @@ frontend public
use_backend node_socket if { path_beg /v3/socket.io/ } use_backend node_socket if { path_beg /v3/socket.io/ }
use_backend node_favicon if { path_beg /rss_feeds/icon/ } use_backend node_favicon if { path_beg /rss_feeds/icon/ }
use_backend node_text if { path_beg /rss_feeds/original_text_fetcher } use_backend node_text if { path_beg /rss_feeds/original_text_fetcher }
use_backend node_page if { path_beg /original_page/ }
use_backend nginx if { path_beg /media/ } use_backend nginx if { path_beg /media/ }
use_backend nginx if { path_beg /static/ } use_backend nginx if { path_beg /static/ }
use_backend nginx if { path_beg /favicon } use_backend nginx if { path_beg /favicon }
@ -65,20 +66,25 @@ frontend public
backend node_socket backend node_socket
http-check expect rstatus 200|503 http-check expect rstatus 200|503
balance roundrobin balance roundrobin
server node_socket node_websocket:8888 check inter 3000ms server node_socket newsblur_node:8008 check inter 3000ms
backend node_favicon backend node_favicon
http-check expect rstatus 200|301|503 http-check expect rstatus 200|301|503
# option httpchk GET /rss_feeds/icon/1 # option httpchk GET /rss_feeds/icon/1
balance roundrobin balance roundrobin
server node_favicon favicons:3030 check inter 3000ms server node_favicon newsblur_node:8008 check inter 3000ms
backend node_text backend node_text
http-check expect rstatus 200|503 http-check expect rstatus 200|503
option httpchk GET /rss_feeds/original_text_fetcher?test=1 option httpchk GET /rss_feeds/original_text_fetcher?test=1
balance roundrobin balance roundrobin
server node_text original_text:4040 check inter 3000ms server node_text newsblur_node:8008 check inter 3000ms
backend node_page
http-check expect rstatus 200|503
option httpchk GET /rss_feeds/original_text_fetcher?test=1
balance roundrobin
server node_page newsblur_node:8008 check inter 3000ms
backend nginx backend nginx
balance roundrobin balance roundrobin

116
node/favicons.coffee
View file

@ -1,69 +1,67 @@
app = require('express')()
server = require('http').Server(app)
mongo = require 'mongodb' mongo = require 'mongodb'
log = require './log.js' log = require './log.js'
favicons = (app) =>
ENV_DEV = process.env.NODE_ENV == 'development'
ENV_PROD = process.env.NODE_ENV == 'production'
ENV_DOCKER = process.env.NODE_ENV == 'docker'
MONGODB_SERVER = "db_mongo"
if ENV_DEV
MONGODB_SERVER = 'localhost'
else if ENV_PROD
MONGODB_SERVER = 'db-mongo.service.nyc1.consul'
MONGODB_PORT = parseInt(process.env.MONGODB_PORT or 27017, 10)
ENV_DEV = process.env.NODE_ENV == 'development' log.debug "Starting NewsBlur Favicon server..."
ENV_PROD = process.env.NODE_ENV == 'production' if !ENV_DEV and !process.env.NODE_ENV
ENV_DOCKER = process.env.NODE_ENV == 'docker' log.debug "Specify NODE_ENV=<development,docker,production>"
MONGODB_SERVER = "db_mongo"
if ENV_DEV
MONGODB_SERVER = 'localhost'
else if ENV_PROD
MONGODB_SERVER = 'db-mongo.service.nyc1.consul'
MONGODB_PORT = parseInt(process.env.MONGODB_PORT or 27017, 10)
log.debug "Starting NewsBlur Favicon server..."
if !ENV_DEV and !process.env.NODE_ENV
log.debug "Specify NODE_ENV=<development,docker,production>"
return
else if ENV_DEV
log.debug "Running as development server"
else if ENV_DOCKER
log.debug "Running as docker server"
else
log.debug "Running as production server"
if ENV_DEV or ENV_DOCKER
url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur"
else
url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred"
do ->
try
client = mongo.MongoClient url, useUnifiedTopology: true
await client.connect()
catch err
log.debug "Error connecting to Mongo: #{err}"
return return
else if ENV_DEV
log.debug "Running as development server"
else if ENV_DOCKER
log.debug "Running as docker server"
else
log.debug "Running as production server"
db = client.db "newsblur" if ENV_DEV or ENV_DOCKER
collection = db.collection "feed_icons" url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur"
else
url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred"
log.debug "Connected to #{db?.serverConfig.s.seedlist[0].host}:#{db?.serverConfig.s.seedlist[0].port}" do ->
if err try
log.debug " ***> Error connecting: #{err}" client = mongo.MongoClient url, useUnifiedTopology: true
await client.connect()
catch err
log.debug "Error connecting to Mongo: #{err}"
return
app.get /\/rss_feeds\/icon\/(\d+)\/?/, (req, res) => db = client.db "newsblur"
feed_id = parseInt(req.params[0], 10) collection = db.collection "feed_icons"
etag = req.header('If-None-Match')
log.debug "Feed: #{feed_id} " + if etag then " / #{etag}" else "" log.debug "Connected to #{db?.serverConfig.s.seedlist[0].host}:#{db?.serverConfig.s.seedlist[0].port}"
collection.findOne _id: feed_id, (err, docs) -> if err
if not err and etag and docs and docs?.color == etag log.debug " ***> Error connecting: #{err}"
log.debug "Cached: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else ""
res.sendStatus 304 app.get /\/rss_feeds\/icon\/(\d+)\/?/, (req, res) =>
else if not err and docs and docs.data feed_id = parseInt(req.params[0], 10)
log.debug "Req: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else "" etag = req.header('If-None-Match')
res.header 'etag', docs.color log.debug "Feed: #{feed_id} " + if etag then " / #{etag}" else ""
body = new Buffer(docs.data, 'base64') collection.findOne _id: feed_id, (err, docs) ->
res.set("Content-Type", "image/png") if not err and etag and docs and docs?.color == etag
res.status(200).send body log.debug "Cached: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else ""
else res.sendStatus 304
log.debug "Redirect: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else "" else if not err and docs and docs.data
if ENV_DEV log.debug "Req: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else ""
res.redirect '/media/img/icons/circular/world.png' res.header 'etag', docs.color
body = new Buffer(docs.data, 'base64')
res.set("Content-Type", "image/png")
res.status(200).send body
else else
res.redirect 'https://www.newsblur.com/media/img/icons/circular/world.png' log.debug "Redirect: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else ""
if ENV_DEV
res.redirect '/media/img/icons/circular/world.png'
else
res.redirect 'https://www.newsblur.com/media/img/icons/circular/world.png'
app.listen 3030 exports.favicons = favicons

155
node/favicons.js
View file

@ -1,97 +1,88 @@
// Generated by CoffeeScript 2.5.1 // Generated by CoffeeScript 2.5.1
(function() { (function() {
var ENV_DEV, ENV_DOCKER, ENV_PROD, MONGODB_PORT, MONGODB_SERVER, app, log, mongo, server, url; var favicons, log, mongo;
app = require('express')();
server = require('http').Server(app);
mongo = require('mongodb'); mongo = require('mongodb');
log = require('./log.js'); log = require('./log.js');
ENV_DEV = process.env.NODE_ENV === 'development'; favicons = (app) => {
var ENV_DEV, ENV_DOCKER, ENV_PROD, MONGODB_PORT, MONGODB_SERVER, url;
ENV_PROD = process.env.NODE_ENV === 'production'; ENV_DEV = process.env.NODE_ENV === 'development';
ENV_PROD = process.env.NODE_ENV === 'production';
ENV_DOCKER = process.env.NODE_ENV === 'docker'; ENV_DOCKER = process.env.NODE_ENV === 'docker';
MONGODB_SERVER = "db_mongo";
MONGODB_SERVER = "db_mongo"; if (ENV_DEV) {
MONGODB_SERVER = 'localhost';
if (ENV_DEV) { } else if (ENV_PROD) {
MONGODB_SERVER = 'localhost'; MONGODB_SERVER = 'db-mongo.service.nyc1.consul';
} else if (ENV_PROD) { }
MONGODB_SERVER = 'db-mongo.service.nyc1.consul'; MONGODB_PORT = parseInt(process.env.MONGODB_PORT || 27017, 10);
} log.debug("Starting NewsBlur Favicon server...");
if (!ENV_DEV && !process.env.NODE_ENV) {
MONGODB_PORT = parseInt(process.env.MONGODB_PORT || 27017, 10); log.debug("Specify NODE_ENV=<development,docker,production>");
log.debug("Starting NewsBlur Favicon server...");
if (!ENV_DEV && !process.env.NODE_ENV) {
log.debug("Specify NODE_ENV=<development,docker,production>");
return;
} else if (ENV_DEV) {
log.debug("Running as development server");
} else if (ENV_DOCKER) {
log.debug("Running as docker server");
} else {
log.debug("Running as production server");
}
if (ENV_DEV || ENV_DOCKER) {
url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur`;
} else {
url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred`;
}
(async function() {
var client, collection, db, err;
try {
client = mongo.MongoClient(url, {
useUnifiedTopology: true
});
await client.connect();
} catch (error) {
err = error;
log.debug(`Error connecting to Mongo: ${err}`);
return; return;
} else if (ENV_DEV) {
log.debug("Running as development server");
} else if (ENV_DOCKER) {
log.debug("Running as docker server");
} else {
log.debug("Running as production server");
} }
db = client.db("newsblur"); if (ENV_DEV || ENV_DOCKER) {
collection = db.collection("feed_icons"); url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur`;
log.debug(`Connected to ${db != null ? db.serverConfig.s.seedlist[0].host : void 0}:${db != null ? db.serverConfig.s.seedlist[0].port : void 0}`); } else {
if (err) { url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred`;
log.debug(` ***> Error connecting: ${err}`);
} }
app.get(/\/rss_feeds\/icon\/(\d+)\/?/, (req, res) => { return (async function() {
var etag, feed_id; var client, collection, db, err;
feed_id = parseInt(req.params[0], 10); try {
etag = req.header('If-None-Match'); client = mongo.MongoClient(url, {
log.debug(`Feed: ${feed_id} ` + (etag ? ` / ${etag}` : "")); useUnifiedTopology: true
return collection.findOne({ });
_id: feed_id await client.connect();
}, function(err, docs) { } catch (error) {
var body; err = error;
if (!err && etag && docs && (docs != null ? docs.color : void 0) === etag) { log.debug(`Error connecting to Mongo: ${err}`);
log.debug(`Cached: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : "")); return;
return res.sendStatus(304); }
} else if (!err && docs && docs.data) { db = client.db("newsblur");
log.debug(`Req: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : "")); collection = db.collection("feed_icons");
res.header('etag', docs.color); log.debug(`Connected to ${db != null ? db.serverConfig.s.seedlist[0].host : void 0}:${db != null ? db.serverConfig.s.seedlist[0].port : void 0}`);
body = new Buffer(docs.data, 'base64'); if (err) {
res.set("Content-Type", "image/png"); log.debug(` ***> Error connecting: ${err}`);
return res.status(200).send(body); }
} else { return app.get(/\/rss_feeds\/icon\/(\d+)\/?/, (req, res) => {
log.debug(`Redirect: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : "")); var etag, feed_id;
if (ENV_DEV) { feed_id = parseInt(req.params[0], 10);
return res.redirect('/media/img/icons/circular/world.png'); etag = req.header('If-None-Match');
log.debug(`Feed: ${feed_id} ` + (etag ? ` / ${etag}` : ""));
return collection.findOne({
_id: feed_id
}, function(err, docs) {
var body;
if (!err && etag && docs && (docs != null ? docs.color : void 0) === etag) {
log.debug(`Cached: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : ""));
return res.sendStatus(304);
} else if (!err && docs && docs.data) {
log.debug(`Req: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : ""));
res.header('etag', docs.color);
body = new Buffer(docs.data, 'base64');
res.set("Content-Type", "image/png");
return res.status(200).send(body);
} else { } else {
return res.redirect('https://www.newsblur.com/media/img/icons/circular/world.png'); log.debug(`Redirect: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : ""));
if (ENV_DEV) {
return res.redirect('/media/img/icons/circular/world.png');
} else {
return res.redirect('https://www.newsblur.com/media/img/icons/circular/world.png');
}
} }
} });
}); });
}); })();
return app.listen(3030); };
})();
exports.favicons = favicons;
}).call(this); }).call(this);

16
node/newsblur.coffee Normal file
View file

@ -0,0 +1,16 @@
app = require('express')()
server = require('http').Server(app)
log = require './log.js'
original_page = require('./original_page.js').original_page
original_text = require('./original_text.js').original_text
favicons = require('./favicons.js').favicons
unread_counts = require('./unread_counts.js').unread_counts
original_page(app)
original_text(app)
favicons(app)
unread_counts(server)
log.debug "Starting NewsBlur Node Server"
app.listen(8008)

31
node/newsblur.js Normal file
View file

@ -0,0 +1,31 @@
// Generated by CoffeeScript 2.5.1
(function() {
var app, favicons, log, original_page, original_text, server, unread_counts;
app = require('express')();
server = require('http').Server(app);
log = require('./log.js');
original_page = require('./original_page.js').original_page;
original_text = require('./original_text.js').original_text;
favicons = require('./favicons.js').favicons;
unread_counts = require('./unread_counts.js').unread_counts;
original_page(app);
original_text(app);
favicons(app);
unread_counts(server);
log.debug("Starting NewsBlur Node Server");
app.listen(8008);
}).call(this);

111
node/original_page.coffee
View file

@ -1,69 +1,70 @@
app = require('express')()
server = require('http').Server(app)
path = require 'path' path = require 'path'
busboy = require 'connect-busboy' busboy = require 'connect-busboy'
fs = require 'fs' fs = require 'fs'
mkdirp = require 'mkdirp' mkdirp = require 'mkdirp'
log = require './log.js'
DEV = process.env.NODE_ENV == 'development' original_page = (app) =>
DEV = process.env.NODE_ENV == 'development'
DB_PATH = if DEV then 'originals' else '/srv/originals' DB_PATH = if DEV then 'originals' else '/srv/originals'
server.listen 3060 app.use busboy()
app.use busboy() app.get /^\/original_page\/(\d+)\/?/, (req, res) =>
if req.query.test
return res.end "OK"
app.get /^\/original_page\/(\d+)\/?/, (req, res) => feedId = parseInt(req.params[0], 10)
if req.query.test etag = req.header('If-None-Match')
return res.end "OK" lastModified = req.header('If-Modified-Since')
feedIdDir = splitFeedId feedId
feedId = parseInt(req.params[0], 10)
etag = req.header('If-None-Match')
lastModified = req.header('If-Modified-Since')
feedIdDir = splitFeedId feedId
filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
fs.exists filePath, (exists, err) ->
console.log " ---> Loading: #{feedId} (#{filePath}). " +
"#{if exists then "" else "NOT FOUND"}"
if not exists
return res.send 404
fs.stat filePath, (err, stats) ->
if not err and etag and stats.mtime == etag
return res.send 304
if not err and lastModified and stats.mtime == lastModified
return res.send 304
fs.readFile filePath, (err, content) ->
res.header 'Etag', Date.parse(stats.mtime)
res.send content
app.post /^\/original_page\/(\d+)\/?/, (req, res) =>
feedId = parseInt(req.params[0], 10)
feedIdDir = splitFeedId feedId
req.pipe req.busboy
req.busboy.on 'file', (fieldname, file, filename) ->
# console.log "Uploading #{fieldname} / #{file} / #{filename}"
filePath = "#{DB_PATH}/#{feedIdDir}.zhtml" filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
filePathDir = path.dirname filePath
mkdirp filePathDir, (err) -> fs.exists filePath, (exists, err) ->
console.log err if err log.debug "Loading: #{feedId} (#{filePath}). " +
fstream = fs.createWriteStream filePath "#{if exists then "" else "NOT FOUND"}"
file.pipe fstream if not exists
fstream.on 'close', -> return res.send 404
fs.stat filePath, (err, stats) -> fs.stat filePath, (err, stats) ->
console.log err if err if not err and etag and stats.mtime == etag
console.log " ---> Saving: #{feedId} (#{filePath}) #{stats.size} bytes" return res.send 304
res.send "OK" if not err and lastModified and stats.mtime == lastModified
return res.send 304
fs.readFile filePath, (err, content) ->
res.header 'Etag', Date.parse(stats.mtime)
res.send content
splitFeedId = (feedId) -> app.post /^\/original_page\/(\d+)\/?/, (req, res) =>
feedId += '' feedId = parseInt(req.params[0], 10)
# x2 = if feedId.length > 1 then '.' + feedId[1] else '' feedIdDir = splitFeedId feedId
rgx = /(\d+)(\d{3})/ req.pipe req.busboy
feedId = feedId.replace rgx, '$1' + '/' + '$2' while rgx.test(feedId) req.busboy.on 'file', (fieldname, file, filename) ->
return feedId; # log.debug "Uploading #{fieldname} / #{file} / #{filename}"
filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
filePathDir = path.dirname filePath
mkdirp filePathDir, (err) ->
log.debug err if err
fstream = fs.createWriteStream filePath
file.pipe fstream
fstream.on 'close', ->
fs.stat filePath, (err, stats) ->
log.debug err if err
log.debug "Saving: #{feedId} (#{filePath}) #{stats.size} bytes"
res.send "OK"
console.log " ---> Starting Original Page server #{if DEV then "on DEV" else "in production"}" splitFeedId = (feedId) ->
feedId += ''
# x2 = if feedId.length > 1 then '.' + feedId[1] else ''
rgx = /(\d+)(\d{3})/
feedId = feedId.replace rgx, '$1' + '/' + '$2' while rgx.test(feedId)
return feedId;
log.debug "Starting Original Page server #{if DEV then "on DEV" else "in production"}"
exports.original_page = original_page

145
node/original_page.js
View file

@ -1,10 +1,6 @@
// Generated by CoffeeScript 2.5.1 // Generated by CoffeeScript 2.5.1
(function() { (function() {
var DB_PATH, DEV, app, busboy, fs, mkdirp, path, server, splitFeedId; var busboy, fs, log, mkdirp, original_page, path;
app = require('express')();
server = require('http').Server(app);
path = require('path'); path = require('path');
@ -14,85 +10,84 @@
mkdirp = require('mkdirp'); mkdirp = require('mkdirp');
DEV = process.env.NODE_ENV === 'development'; log = require('./log.js');
DB_PATH = DEV ? 'originals' : '/srv/originals'; original_page = (app) => {
var DB_PATH, DEV, splitFeedId;
server.listen(3060); DEV = process.env.NODE_ENV === 'development';
DB_PATH = DEV ? 'originals' : '/srv/originals';
app.use(busboy()); app.use(busboy());
app.get(/^\/original_page\/(\d+)\/?/, (req, res) => {
app.get(/^\/original_page\/(\d+)\/?/, (req, res) => { var etag, feedId, feedIdDir, filePath, lastModified;
var etag, feedId, feedIdDir, filePath, lastModified; if (req.query.test) {
if (req.query.test) { return res.end("OK");
return res.end("OK");
}
feedId = parseInt(req.params[0], 10);
etag = req.header('If-None-Match');
lastModified = req.header('If-Modified-Since');
feedIdDir = splitFeedId(feedId);
filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
return fs.exists(filePath, function(exists, err) {
console.log(` ---> Loading: ${feedId} (${filePath}). ` + `${exists ? "" : "NOT FOUND"}`);
if (!exists) {
return res.send(404);
} }
return fs.stat(filePath, function(err, stats) { feedId = parseInt(req.params[0], 10);
if (!err && etag && stats.mtime === etag) { etag = req.header('If-None-Match');
return res.send(304); lastModified = req.header('If-Modified-Since');
} feedIdDir = splitFeedId(feedId);
if (!err && lastModified && stats.mtime === lastModified) {
return res.send(304);
}
return fs.readFile(filePath, function(err, content) {
res.header('Etag', Date.parse(stats.mtime));
return res.send(content);
});
});
});
});
app.post(/^\/original_page\/(\d+)\/?/, (req, res) => {
var feedId, feedIdDir;
feedId = parseInt(req.params[0], 10);
feedIdDir = splitFeedId(feedId);
req.pipe(req.busboy);
return req.busboy.on('file', function(fieldname, file, filename) {
var filePath, filePathDir;
// console.log "Uploading #{fieldname} / #{file} / #{filename}"
filePath = `${DB_PATH}/${feedIdDir}.zhtml`; filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
filePathDir = path.dirname(filePath); return fs.exists(filePath, function(exists, err) {
return mkdirp(filePathDir, function(err) { log.debug(`Loading: ${feedId} (${filePath}). ` + `${exists ? "" : "NOT FOUND"}`);
var fstream; if (!exists) {
if (err) { return res.send(404);
console.log(err);
} }
fstream = fs.createWriteStream(filePath); return fs.stat(filePath, function(err, stats) {
file.pipe(fstream); if (!err && etag && stats.mtime === etag) {
return fstream.on('close', function() { return res.send(304);
return fs.stat(filePath, function(err, stats) { }
if (err) { if (!err && lastModified && stats.mtime === lastModified) {
console.log(err); return res.send(304);
} }
console.log(` ---> Saving: ${feedId} (${filePath}) ${stats.size} bytes`); return fs.readFile(filePath, function(err, content) {
return res.send("OK"); res.header('Etag', Date.parse(stats.mtime));
return res.send(content);
}); });
}); });
}); });
}); });
}); app.post(/^\/original_page\/(\d+)\/?/, (req, res) => {
var feedId, feedIdDir;
splitFeedId = function(feedId) { feedId = parseInt(req.params[0], 10);
var rgx; feedIdDir = splitFeedId(feedId);
feedId += ''; req.pipe(req.busboy);
// x2 = if feedId.length > 1 then '.' + feedId[1] else '' return req.busboy.on('file', function(fieldname, file, filename) {
rgx = /(\d+)(\d{3})/; var filePath, filePathDir;
while (rgx.test(feedId)) { // log.debug "Uploading #{fieldname} / #{file} / #{filename}"
feedId = feedId.replace(rgx, '$1' + '/' + '$2'); filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
} filePathDir = path.dirname(filePath);
return feedId; return mkdirp(filePathDir, function(err) {
var fstream;
if (err) {
log.debug(err);
}
fstream = fs.createWriteStream(filePath);
file.pipe(fstream);
return fstream.on('close', function() {
return fs.stat(filePath, function(err, stats) {
if (err) {
log.debug(err);
}
log.debug(`Saving: ${feedId} (${filePath}) ${stats.size} bytes`);
return res.send("OK");
});
});
});
});
});
splitFeedId = function(feedId) {
var rgx;
feedId += '';
// x2 = if feedId.length > 1 then '.' + feedId[1] else ''
rgx = /(\d+)(\d{3})/;
while (rgx.test(feedId)) {
feedId = feedId.replace(rgx, '$1' + '/' + '$2');
}
return feedId;
};
return log.debug(`Starting Original Page server ${DEV ? "on DEV" : "in production"}`);
}; };
console.log(` ---> Starting Original Page server ${DEV ? "on DEV" : "in production"}`); exports.original_page = original_page;
}).call(this); }).call(this);

53
node/original_text.coffee
View file

@ -1,36 +1,35 @@
Mercury = require('@postlight/mercury-parser') Mercury = require('@postlight/mercury-parser')
app = require('express')()
server = require('http').Server(app)
log = require './log.js' log = require './log.js'
DEV = process.env.NODE_ENV == 'development' original_text = (app) =>
DEV = process.env.NODE_ENV == 'development'
log.debug "Starting NewsBlur Original Text Fetcher / Mercury Parser..." log.debug "Starting NewsBlur Original Text Fetcher / Mercury Parser..."
if !DEV and !process.env.NODE_ENV if !DEV and !process.env.NODE_ENV
log.debug "Specify NODE_ENV=<development,production>" log.debug "Specify NODE_ENV=<development,production>"
return return
else if DEV else if DEV
log.debug "Running as development server" log.debug "Running as development server"
else else
log.debug "Running as production server" log.debug "Running as production server"
app.get /\/rss_feeds\/original_text_fetcher\/?/, (req, res) => app.get /\/rss_feeds\/original_text_fetcher\/?/, (req, res) =>
res.setHeader 'Content-Type', 'application/json' res.setHeader 'Content-Type', 'application/json'
if req.query.test if req.query.test
return res.end "OK" return res.end "OK"
url = req.query.url url = req.query.url
if !url if !url
log.debug "Missing url" log.debug "Missing url"
return res.end JSON.stringify error: "Missing `url` query parameter." return res.end JSON.stringify error: "Missing `url` query parameter."
api_key = req.header('x-api-key') or req.query.apikey api_key = req.header('x-api-key') or req.query.apikey
if not DEV and (!api_key or api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") == -1) if not DEV and (!api_key or api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") == -1)
log.debug "Mismatched API key: #{url} / #{api_key}" log.debug "Mismatched API key: #{url} / #{api_key}"
return res.end JSON.stringify error: "Invalid API key. You need to set up your own Original Text server." return res.end JSON.stringify error: "Invalid API key. You need to set up your own Original Text server."
Mercury.parse(url).then (result) => Mercury.parse(url).then (result) =>
log.debug "Fetched: #{url}" log.debug "Fetched: #{url}"
res.end JSON.stringify result res.end JSON.stringify result
app.listen 4040 exports.original_text = original_text

80
node/original_text.js
View file

@ -1,54 +1,50 @@
// Generated by CoffeeScript 2.5.1 // Generated by CoffeeScript 2.5.1
(function() { (function() {
var DEV, Mercury, app, log, server; var Mercury, log, original_text;
Mercury = require('@postlight/mercury-parser'); Mercury = require('@postlight/mercury-parser');
app = require('express')();
server = require('http').Server(app);
log = require('./log.js'); log = require('./log.js');
DEV = process.env.NODE_ENV === 'development'; original_text = (app) => {
var DEV;
log.debug("Starting NewsBlur Original Text Fetcher / Mercury Parser..."); DEV = process.env.NODE_ENV === 'development';
log.debug("Starting NewsBlur Original Text Fetcher / Mercury Parser...");
if (!DEV && !process.env.NODE_ENV) { if (!DEV && !process.env.NODE_ENV) {
log.debug("Specify NODE_ENV=<development,production>"); log.debug("Specify NODE_ENV=<development,production>");
return; return;
} else if (DEV) { } else if (DEV) {
log.debug("Running as development server"); log.debug("Running as development server");
} else { } else {
log.debug("Running as production server"); log.debug("Running as production server");
}
app.get(/\/rss_feeds\/original_text_fetcher\/?/, (req, res) => {
var api_key, url;
res.setHeader('Content-Type', 'application/json');
if (req.query.test) {
return res.end("OK");
} }
url = req.query.url; return app.get(/\/rss_feeds\/original_text_fetcher\/?/, (req, res) => {
if (!url) { var api_key, url;
log.debug("Missing url"); res.setHeader('Content-Type', 'application/json');
return res.end(JSON.stringify({ if (req.query.test) {
error: "Missing `url` query parameter." return res.end("OK");
})); }
} url = req.query.url;
api_key = req.header('x-api-key') || req.query.apikey; if (!url) {
if (!DEV && (!api_key || api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") === -1)) { log.debug("Missing url");
log.debug(`Mismatched API key: ${url} / ${api_key}`); return res.end(JSON.stringify({
return res.end(JSON.stringify({ error: "Missing `url` query parameter."
error: "Invalid API key. You need to set up your own Original Text server." }));
})); }
} api_key = req.header('x-api-key') || req.query.apikey;
return Mercury.parse(url).then((result) => { if (!DEV && (!api_key || api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") === -1)) {
log.debug(`Fetched: ${url}`); log.debug(`Mismatched API key: ${url} / ${api_key}`);
return res.end(JSON.stringify(result)); return res.end(JSON.stringify({
error: "Invalid API key. You need to set up your own Original Text server."
}));
}
return Mercury.parse(url).then((result) => {
log.debug(`Fetched: ${url}`);
return res.end(JSON.stringify(result));
});
}); });
}); };
app.listen(4040); exports.original_text = original_text;
}).call(this); }).call(this);

162
node/unread_counts.coffee
View file

@ -2,103 +2,87 @@ fs = require 'fs'
redis = require 'redis' redis = require 'redis'
log = require './log.js' log = require './log.js'
ENV_DEV = process.env.NODE_ENV == 'development' unread_counts = (server) =>
ENV_PROD = process.env.NODE_ENV == 'production' ENV_DEV = process.env.NODE_ENV == 'development'
ENV_DOCKER = process.env.NODE_ENV == 'docker' ENV_PROD = process.env.NODE_ENV == 'production'
REDIS_SERVER = "redis" ENV_DOCKER = process.env.NODE_ENV == 'docker'
if ENV_DEV REDIS_SERVER = "redis"
REDIS_SERVER = 'localhost' if ENV_DEV
else if ENV_PROD REDIS_SERVER = 'localhost'
REDIS_SERVER = 'db-redis-user.service.nyc1.consul' else if ENV_PROD
SECURE = !!process.env.NODE_SSL REDIS_SERVER = 'db-redis-user.service.nyc1.consul'
REDIS_PORT = if ENV_DOCKER then 6579 else 6379 SECURE = !!process.env.NODE_SSL
REDIS_PORT = if ENV_DOCKER then 6579 else 6379
# client = redis.createClient 6379, REDIS_SERVER # client = redis.createClient 6379, REDIS_SERVER
# RedisStore = require 'socket.io/lib/stores/redis' # RedisStore = require 'socket.io/lib/stores/redis'
# rpub = redis.createClient 6379, REDIS_SERVER # rpub = redis.createClient 6379, REDIS_SERVER
# rsub = redis.createClient 6379, REDIS_SERVER # rsub = redis.createClient 6379, REDIS_SERVER
# rclient = redis.createClient 6379, REDIS_SERVER # rclient = redis.createClient 6379, REDIS_SERVER
log.debug "Starting NewsBlur unread count server..." log.debug "Starting NewsBlur unread count server..."
if !ENV_DEV and !process.env.NODE_ENV if !ENV_DEV and !process.env.NODE_ENV
log.debug "Specify NODE_ENV=<development,production>" log.debug "Specify NODE_ENV=<development,production>"
return return
else if ENV_DEV else if ENV_DEV
log.debug "Running as development server" log.debug "Running as development server"
else else
log.debug "Running as production server" log.debug "Running as production server"
io = require('socket.io')(server, path: "/v3/socket.io")
if SECURE # io.set('transports', ['websocket'])
privateKey = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.key').toString()
certificate = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.crt').toString()
# ca = fs.readFileSync('./config/certificates/intermediate.crt').toString()
options =
port: 8889
key: privateKey
cert: certificate
app = require('https').createServer options
io = require('socket.io')(app, path: "/v3/socket.io")
app.listen options.port
log.debug "Listening securely on port #{options.port}"
else
options =
port: 8888
app = require('http').createServer()
io = require('socket.io')(app, path: "/v3/socket.io")
app.listen options.port
log.debug "Listening on port #{options.port}"
# io.set('transports', ['websocket']) # io.set 'store', new RedisStore
# redisPub : rpub
# redisSub : rsub
# redisClient : rclient
# io.set 'store', new RedisStore io.on 'connection', (socket) ->
# redisPub : rpub ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address
# redisSub : rsub
# redisClient : rclient
io.on 'connection', (socket) -> socket.on 'subscribe:feeds', (@feeds, @username) =>
ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address log.info @username, "Connecting (#{@feeds.length} feeds, #{ip})," +
" (#{io.engine.clientsCount} connected) " +
socket.on 'subscribe:feeds', (@feeds, @username) =>
log.info @username, "Connecting (#{@feeds.length} feeds, #{ip})," +
" (#{io.engine.clientsCount} connected) " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}"
if not @username
return
socket.on "error", (err) ->
log.debug "Error (socket): #{err}"
socket.subscribe?.quit()
socket.subscribe = redis.createClient REDIS_PORT, REDIS_SERVER
socket.subscribe.on "error", (err) =>
log.info @username, "Error: #{err} (#{@feeds.length} feeds)"
socket.subscribe?.quit()
socket.subscribe.on "connect", =>
log.info @username, "Connected (#{@feeds.length} feeds, #{ip})," +
" (#{io.engine.clientsCount} connected) " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}"
socket.subscribe.subscribe @feeds
feeds_story = @feeds.map (f) -> "#{f}:story"
socket.subscribe.subscribe feeds_story
socket.subscribe.subscribe @username
socket.subscribe.on 'message', (channel, message) =>
event_name = 'feed:update'
if channel == @username
event_name = 'user:update'
else if channel.indexOf(':story') >= 0
event_name = 'feed:story:new'
log.info @username, "Update on #{channel}: #{event_name} - #{message}"
socket.emit event_name, channel, message
socket.on 'disconnect', () =>
socket.subscribe?.quit()
log.info @username, "Disconnect (#{@feeds?.length} feeds, #{ip})," +
" there are now #{io.engine.clientsCount} users. " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}" " #{if SECURE then "(SSL)" else "(non-SSL)"}"
io.sockets.on 'error', (err) -> if not @username
log.debug "Error (sockets): #{err}" return
socket.on "error", (err) ->
log.debug "Error (socket): #{err}"
socket.subscribe?.quit()
socket.subscribe = redis.createClient REDIS_PORT, REDIS_SERVER
socket.subscribe.on "error", (err) =>
log.info @username, "Error: #{err} (#{@feeds.length} feeds)"
socket.subscribe?.quit()
socket.subscribe.on "connect", =>
log.info @username, "Connected (#{@feeds.length} feeds, #{ip})," +
" (#{io.engine.clientsCount} connected) " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}"
socket.subscribe.subscribe @feeds
feeds_story = @feeds.map (f) -> "#{f}:story"
socket.subscribe.subscribe feeds_story
socket.subscribe.subscribe @username
socket.subscribe.on 'message', (channel, message) =>
event_name = 'feed:update'
if channel == @username
event_name = 'user:update'
else if channel.indexOf(':story') >= 0
event_name = 'feed:story:new'
log.info @username, "Update on #{channel}: #{event_name} - #{message}"
socket.emit event_name, channel, message
socket.on 'disconnect', () =>
socket.subscribe?.quit()
log.info @username, "Disconnect (#{@feeds?.length} feeds, #{ip})," +
" there are now #{io.engine.clientsCount} users. " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}"
io.sockets.on 'error', (err) ->
log.debug "Error (sockets): #{err}"
exports.unread_counts = unread_counts

205
node/unread_counts.js
View file

@ -1,6 +1,6 @@
// Generated by CoffeeScript 2.5.1 // Generated by CoffeeScript 2.5.1
(function() { (function() {
var ENV_DEV, ENV_DOCKER, ENV_PROD, REDIS_PORT, REDIS_SERVER, SECURE, app, certificate, fs, io, log, options, privateKey, redis; var fs, log, redis, unread_counts;
fs = require('fs'); fs = require('fs');
@ -8,130 +8,101 @@
log = require('./log.js'); log = require('./log.js');
ENV_DEV = process.env.NODE_ENV === 'development'; unread_counts = (server) => {
var ENV_DEV, ENV_DOCKER, ENV_PROD, REDIS_PORT, REDIS_SERVER, SECURE, io;
ENV_DEV = process.env.NODE_ENV === 'development';
ENV_PROD = process.env.NODE_ENV === 'production';
ENV_DOCKER = process.env.NODE_ENV === 'docker';
REDIS_SERVER = "redis";
if (ENV_DEV) {
REDIS_SERVER = 'localhost';
} else if (ENV_PROD) {
REDIS_SERVER = 'db-redis-user.service.nyc1.consul';
}
SECURE = !!process.env.NODE_SSL;
REDIS_PORT = ENV_DOCKER ? 6579 : 6379;
// client = redis.createClient 6379, REDIS_SERVER
ENV_PROD = process.env.NODE_ENV === 'production'; // RedisStore = require 'socket.io/lib/stores/redis'
// rpub = redis.createClient 6379, REDIS_SERVER
ENV_DOCKER = process.env.NODE_ENV === 'docker'; // rsub = redis.createClient 6379, REDIS_SERVER
// rclient = redis.createClient 6379, REDIS_SERVER
REDIS_SERVER = "redis"; log.debug("Starting NewsBlur unread count server...");
if (!ENV_DEV && !process.env.NODE_ENV) {
if (ENV_DEV) { log.debug("Specify NODE_ENV=<development,production>");
REDIS_SERVER = 'localhost'; return;
} else if (ENV_PROD) { } else if (ENV_DEV) {
REDIS_SERVER = 'db-redis-user.service.nyc1.consul'; log.debug("Running as development server");
} } else {
log.debug("Running as production server");
SECURE = !!process.env.NODE_SSL; }
io = require('socket.io')(server, {
REDIS_PORT = ENV_DOCKER ? 6579 : 6379;
// client = redis.createClient 6379, REDIS_SERVER
// RedisStore = require 'socket.io/lib/stores/redis'
// rpub = redis.createClient 6379, REDIS_SERVER
// rsub = redis.createClient 6379, REDIS_SERVER
// rclient = redis.createClient 6379, REDIS_SERVER
log.debug("Starting NewsBlur unread count server...");
if (!ENV_DEV && !process.env.NODE_ENV) {
log.debug("Specify NODE_ENV=<development,production>");
return;
} else if (ENV_DEV) {
log.debug("Running as development server");
} else {
log.debug("Running as production server");
}
if (SECURE) {
privateKey = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.key').toString();
certificate = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.crt').toString();
// ca = fs.readFileSync('./config/certificates/intermediate.crt').toString()
options = {
port: 8889,
key: privateKey,
cert: certificate
};
app = require('https').createServer(options);
io = require('socket.io')(app, {
path: "/v3/socket.io" path: "/v3/socket.io"
}); });
app.listen(options.port); // io.set('transports', ['websocket'])
log.debug(`Listening securely on port ${options.port}`);
} else {
options = {
port: 8888
};
app = require('http').createServer();
io = require('socket.io')(app, {
path: "/v3/socket.io"
});
app.listen(options.port);
log.debug(`Listening on port ${options.port}`);
}
// io.set('transports', ['websocket']) // io.set 'store', new RedisStore
// redisPub : rpub
// io.set 'store', new RedisStore // redisSub : rsub
// redisPub : rpub // redisClient : rclient
// redisSub : rsub io.on('connection', function(socket) {
// redisClient : rclient var ip;
io.on('connection', function(socket) { ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address;
var ip; socket.on('subscribe:feeds', (feeds, username) => {
ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address; var ref;
socket.on('subscribe:feeds', (feeds, username) => { this.feeds = feeds;
var ref; this.username = username;
this.feeds = feeds; log.info(this.username, `Connecting (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
this.username = username; if (!this.username) {
log.info(this.username, `Connecting (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`); return;
if (!this.username) {
return;
}
socket.on("error", function(err) {
return log.debug(`Error (socket): ${err}`);
});
if ((ref = socket.subscribe) != null) {
ref.quit();
}
socket.subscribe = redis.createClient(REDIS_PORT, REDIS_SERVER);
socket.subscribe.on("error", (err) => {
var ref1;
log.info(this.username, `Error: ${err} (${this.feeds.length} feeds)`);
return (ref1 = socket.subscribe) != null ? ref1.quit() : void 0;
});
socket.subscribe.on("connect", () => {
var feeds_story;
log.info(this.username, `Connected (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
socket.subscribe.subscribe(this.feeds);
feeds_story = this.feeds.map(function(f) {
return `${f}:story`;
});
socket.subscribe.subscribe(feeds_story);
return socket.subscribe.subscribe(this.username);
});
return socket.subscribe.on('message', (channel, message) => {
var event_name;
event_name = 'feed:update';
if (channel === this.username) {
event_name = 'user:update';
} else if (channel.indexOf(':story') >= 0) {
event_name = 'feed:story:new';
} }
log.info(this.username, `Update on ${channel}: ${event_name} - ${message}`); socket.on("error", function(err) {
return socket.emit(event_name, channel, message); return log.debug(`Error (socket): ${err}`);
});
if ((ref = socket.subscribe) != null) {
ref.quit();
}
socket.subscribe = redis.createClient(REDIS_PORT, REDIS_SERVER);
socket.subscribe.on("error", (err) => {
var ref1;
log.info(this.username, `Error: ${err} (${this.feeds.length} feeds)`);
return (ref1 = socket.subscribe) != null ? ref1.quit() : void 0;
});
socket.subscribe.on("connect", () => {
var feeds_story;
log.info(this.username, `Connected (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
socket.subscribe.subscribe(this.feeds);
feeds_story = this.feeds.map(function(f) {
return `${f}:story`;
});
socket.subscribe.subscribe(feeds_story);
return socket.subscribe.subscribe(this.username);
});
return socket.subscribe.on('message', (channel, message) => {
var event_name;
event_name = 'feed:update';
if (channel === this.username) {
event_name = 'user:update';
} else if (channel.indexOf(':story') >= 0) {
event_name = 'feed:story:new';
}
log.info(this.username, `Update on ${channel}: ${event_name} - ${message}`);
return socket.emit(event_name, channel, message);
});
});
return socket.on('disconnect', () => {
var ref, ref1;
if ((ref = socket.subscribe) != null) {
ref.quit();
}
return log.info(this.username, `Disconnect (${(ref1 = this.feeds) != null ? ref1.length : void 0} feeds, ${ip}),` + ` there are now ${io.engine.clientsCount} users. ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
}); });
}); });
return socket.on('disconnect', () => { return io.sockets.on('error', function(err) {
var ref, ref1; return log.debug(`Error (sockets): ${err}`);
if ((ref = socket.subscribe) != null) {
ref.quit();
}
return log.info(this.username, `Disconnect (${(ref1 = this.feeds) != null ? ref1.length : void 0} feeds, ${ip}),` + ` there are now ${io.engine.clientsCount} users. ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`);
}); });
}); };
io.sockets.on('error', function(err) { exports.unread_counts = unread_counts;
return log.debug(`Error (sockets): ${err}`);
});
}).call(this); }).call(this);

4
templates/analyzer/popularity_query.xhtml
View file

@ -12,12 +12,12 @@
<div class="NB-static-form-wrapper"> <div class="NB-static-form-wrapper">
<div class="NB-delete-form NB-static-form"> <div class="NB-delete-form NB-static-form">
<h2>NewsBlur Publisher Keyword Popularity XLSX Creator for YC Founders</h2> <h2>NewsBlur News Popularity Spreadsheet Creator</h2>
<h5>Search for topics across millions of stories from millions of publishers</h5> <h5>Search for topics across millions of stories from millions of publishers</h5>
{% if success %} {% if success %}
<h4 style="color: #1C6130; text-align: center">Got it!<br>Email should be sent within the hour.</h4> <h4 style="color: #1C6130; text-align: center">Got it!<br>Email should be sent within the next few minutes.</h4>
{% else %} {% else %}
<form action="" method="POST">{% csrf_token %} <form action="" method="POST">{% csrf_token %}
<div class="NB-fields"> <div class="NB-fields">