From 2906e924996bb0291e281d840421494099876da7 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 16 Mar 2021 19:34:11 -0400 Subject: [PATCH] Merging all four node servers into a single one on port 8008. Updating HAProxy on both consul and docker-compose. --- ansible/roles/node/tasks/main.yml | 45 ++-- .../roles/node/templates/consul_service.json | 15 +- apps/rss_feeds/models.py | 2 +- docker-compose.yml | 44 +--- docker/haproxy/haproxy.consul.cfg.j2 | 8 +- docker/haproxy/haproxy.docker-compose.cfg | 12 +- node/favicons.coffee | 116 +++++----- node/favicons.js | 155 +++++++------ node/newsblur.coffee | 16 ++ node/newsblur.js | 31 +++ node/original_page.coffee | 113 +++++----- node/original_page.js | 145 ++++++------- node/original_text.coffee | 57 +++-- node/original_text.js | 80 ++++--- node/unread_counts.coffee | 166 +++++++------- node/unread_counts.js | 205 ++++++++---------- templates/analyzer/popularity_query.xhtml | 6 +- 17 files changed, 578 insertions(+), 638 deletions(-) create mode 100644 node/newsblur.coffee create mode 100644 node/newsblur.js diff --git a/ansible/roles/node/tasks/main.yml b/ansible/roles/node/tasks/main.yml index da1ecb2c7..83b0faf5a 100644 --- a/ansible/roles/node/tasks/main.yml +++ b/ansible/roles/node/tasks/main.yml @@ -2,10 +2,10 @@ - name: Start node docker containers become: yes docker_container: - name: "{{ item.container_name }}" + name: node image: newsblur/newsblur_node state: started - command: "{{ item.command }}" + command: node /srv/node/newsblur.js container_default_behavior: no_defaults pull: true network_mode: host @@ -15,23 +15,11 @@ volumes: - /srv/newsblur/node:/srv/node with_items: - - container_name: node_socket - ports: 8888:8888 - command: node /srv/node/unread_counts.js - target_host: node-socket - - container_name: original_pages - ports: 3060:3060 - command: node /srv/node/original_page.js - target_host: node-page - - container_name: original_text - ports: 4040:4040 - command: node /srv/node/original_text.js - target_host: node-text - - container_name: favicons - ports: 3030:3030 - command: node /srv/node/favicons.js - target_host: node-favicons - when: item.target_host in inventory_hostname + - node-socket + - node-page + - node-text + - node-favicons + when: item in inventory_hostname - name: Start non-newsblur node docker containers become: yes @@ -59,18 +47,13 @@ become: yes template: src: consul_service.json - dest: /etc/consul.d/{{item.target_host}}.json + dest: /etc/consul.d/{{item}}.json with_items: - - port: 8888 - target_host: node-socket - - port: 3060 - target_host: node-page - - port: 4040 - target_host: node-text - - port: 3030 - target_host: node-favicons - - port: 8080 - target_host: node-images + - node-socket + - node-page + - node-text + - node-favicons + - node-images notify: - reload consul - when: item.target_host in inventory_hostname and disable_consul_services is not defined + when: item in inventory_hostname and disable_consul_services is not defined diff --git a/ansible/roles/node/templates/consul_service.json b/ansible/roles/node/templates/consul_service.json index e0b98a8be..746399fd0 100644 --- a/ansible/roles/node/templates/consul_service.json +++ b/ansible/roles/node/templates/consul_service.json @@ -1,11 +1,12 @@ { "service": { - "name": "{{ item.target_host }}", - "id": "{{ inventory_hostname }}", + "name": "{{ inventory_hostname|regex_replace('\d+', '') }}", + "id": "{{ inventory_hostname }}-web", "tags": [ - "node" + "web" ], - "port": {{ item.port }} + "port": 8008 + {% if item.target_host in ["node-images", "node-favicons", "node-text", "node-page"] %} , "checks": [{ @@ -13,11 +14,11 @@ {% if item.target_host == "node-images" %} "http": "http://{{ ansible_ssh_host }}:8080/sc,sN1megONJiGNy-CCvqzVPTv-TWRhgSKhFlf61XAYESl4=/http:/samuelclay.com/static/images/2019%20-%20Cuba.jpg", {% elif item.target_host == "node-favicons" %} - "http": "http://{{ ansible_ssh_host }}:3030/rss_feeds/icon/1", + "http": "http://{{ ansible_ssh_host }}:8008/rss_feeds/icon/1", {% elif item.target_host == "node-text" %} - "http": "http://{{ ansible_ssh_host }}:4040/rss_feeds/original_text_fetcher?test=1", + "http": "http://{{ ansible_ssh_host }}:8008/rss_feeds/original_text_fetcher?test=1", {% elif item.target_host == "node-page" %} - "http": "http://{{ ansible_ssh_host }}:3060/original_page/1?test=1", + "http": "http://{{ ansible_ssh_host }}:8008/original_page/1?test=1", {% endif %} "interval": "15s" }] diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index a92d7d3d0..a8b238da9 100755 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1688,10 +1688,10 @@ class Feed(models.Model): return sorted_popularity def well_read_score(self): + """Average percentage of stories read vs published across recently active subscribers""" from apps.reader.models import UserSubscription from apps.social.models import MSharedStory - # Average percentage of stories read vs published across recently active subscribers r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL) p = r.pipeline() diff --git a/docker-compose.yml b/docker-compose.yml index af65027e6..f1586e370 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,51 +92,18 @@ services: volumes: - ./docker/volumes/db_mongo:/data/db - original_pages: + newsblur_node: image: newsblur/newsblur_node - container_name: node_original_page + container_name: node environment: - NODE_ENV=development - command: node original_page.js - ports: - - 3060:3060 - volumes: - - node-files:/srv - - original_text: - image: newsblur/newsblur_node - container_name: node_original_text - environment: - - NODE_ENV=development - ports: - - 4040:4040 - command: node original_text.js - volumes: - - node-files:/srv - - favicons: - image: newsblur/newsblur_node - container_name: node_favicons - environment: - - NODE_ENV=docker - MONGODB_PORT=29019 - ports: - - 3030:3030 - command: node favicons.js + command: node newsblur.js + stop_signal: HUP depends_on: - db_postgres - volumes: - - node-files:/srv - - node_websocket: - container_name: node_websocket - image: newsblur/newsblur_node - environment: - - NODE_ENV=docker - command: node unread_counts.js - stop_signal: HUP ports: - - 8888:8888 + - 8008:8008 volumes: - node-files:/srv @@ -167,6 +134,7 @@ services: depends_on: - nginx - newsblur_web + - newsblur_node - db_redis - db_postgres - db_elasticsearch diff --git a/docker/haproxy/haproxy.consul.cfg.j2 b/docker/haproxy/haproxy.consul.cfg.j2 index b458336a7..d564c1ae2 100644 --- a/docker/haproxy/haproxy.consul.cfg.j2 +++ b/docker/haproxy/haproxy.consul.cfg.j2 @@ -114,26 +114,26 @@ backend node_socket balance roundrobin default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none {% for host in groups.node_socket %} - server {{host}} {{host}}.node.nyc1.consul:8888 + server {{host}} {{host}}.node.nyc1.consul:8008 {% endfor %} backend node_favicon http-check expect rstatus 200|503 option httpchk GET /rss_feeds/icon/1 balance roundrobin - server-template node-favicons 1 _node-favicons._tcp.service.nyc1.consul:3030 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + server-template node-favicons 1 _node-favicons._tcp.service.nyc1.consul:8008 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none backend node_text http-check expect rstatus 200|503 option httpchk GET /rss_feeds/original_text_fetcher?test=1 balance roundrobin - server-template node-text 1 _node-text._tcp.service.nyc1.consul:4040 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + server-template node-text 1 _node-text._tcp.service.nyc1.consul:8008 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none backend node_page http-check expect rstatus 200|503 option httpchk GET /original_page/1?test=1 balance roundrobin - server-template node-page 1 _node-page._tcp.service.nyc1.consul:3060 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + server-template node-page 1 _node-page._tcp.service.nyc1.consul:8008 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none backend postgres option httpchk GET /db_check/postgres diff --git a/docker/haproxy/haproxy.docker-compose.cfg b/docker/haproxy/haproxy.docker-compose.cfg index 691c6ce9a..4b30d334e 100644 --- a/docker/haproxy/haproxy.docker-compose.cfg +++ b/docker/haproxy/haproxy.docker-compose.cfg @@ -51,6 +51,7 @@ frontend public use_backend node_socket if { path_beg /v3/socket.io/ } use_backend node_favicon if { path_beg /rss_feeds/icon/ } use_backend node_text if { path_beg /rss_feeds/original_text_fetcher } + use_backend node_page if { path_beg /original_page/ } use_backend nginx if { path_beg /media/ } use_backend nginx if { path_beg /static/ } use_backend nginx if { path_beg /favicon } @@ -65,20 +66,25 @@ frontend public backend node_socket http-check expect rstatus 200|503 balance roundrobin - server node_socket node_websocket:8888 check inter 3000ms + server node_socket newsblur_node:8008 check inter 3000ms backend node_favicon http-check expect rstatus 200|301|503 # option httpchk GET /rss_feeds/icon/1 balance roundrobin - server node_favicon favicons:3030 check inter 3000ms + server node_favicon newsblur_node:8008 check inter 3000ms backend node_text http-check expect rstatus 200|503 option httpchk GET /rss_feeds/original_text_fetcher?test=1 balance roundrobin - server node_text original_text:4040 check inter 3000ms + server node_text newsblur_node:8008 check inter 3000ms +backend node_page + http-check expect rstatus 200|503 + option httpchk GET /rss_feeds/original_text_fetcher?test=1 + balance roundrobin + server node_page newsblur_node:8008 check inter 3000ms backend nginx balance roundrobin diff --git a/node/favicons.coffee b/node/favicons.coffee index 95f9e08df..7a9e4afc3 100644 --- a/node/favicons.coffee +++ b/node/favicons.coffee @@ -1,69 +1,67 @@ -app = require('express')() -server = require('http').Server(app) mongo = require 'mongodb' log = require './log.js' +favicons = (app) => + ENV_DEV = process.env.NODE_ENV == 'development' + ENV_PROD = process.env.NODE_ENV == 'production' + ENV_DOCKER = process.env.NODE_ENV == 'docker' + MONGODB_SERVER = "db_mongo" + if ENV_DEV + MONGODB_SERVER = 'localhost' + else if ENV_PROD + MONGODB_SERVER = 'db-mongo.service.nyc1.consul' + MONGODB_PORT = parseInt(process.env.MONGODB_PORT or 27017, 10) -ENV_DEV = process.env.NODE_ENV == 'development' -ENV_PROD = process.env.NODE_ENV == 'production' -ENV_DOCKER = process.env.NODE_ENV == 'docker' -MONGODB_SERVER = "db_mongo" -if ENV_DEV - MONGODB_SERVER = 'localhost' -else if ENV_PROD - MONGODB_SERVER = 'db-mongo.service.nyc1.consul' -MONGODB_PORT = parseInt(process.env.MONGODB_PORT or 27017, 10) - -log.debug "Starting NewsBlur Favicon server..." -if !ENV_DEV and !process.env.NODE_ENV - log.debug "Specify NODE_ENV=" - return -else if ENV_DEV - log.debug "Running as development server" -else if ENV_DOCKER - log.debug "Running as docker server" -else - log.debug "Running as production server" - -if ENV_DEV or ENV_DOCKER - url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur" -else - url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred" - -do -> - try - client = mongo.MongoClient url, useUnifiedTopology: true - await client.connect() - catch err - log.debug "Error connecting to Mongo: #{err}" + log.debug "Starting NewsBlur Favicon server..." + if !ENV_DEV and !process.env.NODE_ENV + log.debug "Specify NODE_ENV=" return + else if ENV_DEV + log.debug "Running as development server" + else if ENV_DOCKER + log.debug "Running as docker server" + else + log.debug "Running as production server" + + if ENV_DEV or ENV_DOCKER + url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur" + else + url = "mongodb://#{MONGODB_SERVER}:#{MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred" - db = client.db "newsblur" - collection = db.collection "feed_icons" + do -> + try + client = mongo.MongoClient url, useUnifiedTopology: true + await client.connect() + catch err + log.debug "Error connecting to Mongo: #{err}" + return - log.debug "Connected to #{db?.serverConfig.s.seedlist[0].host}:#{db?.serverConfig.s.seedlist[0].port}" - if err - log.debug " ***> Error connecting: #{err}" + db = client.db "newsblur" + collection = db.collection "feed_icons" - app.get /\/rss_feeds\/icon\/(\d+)\/?/, (req, res) => - feed_id = parseInt(req.params[0], 10) - etag = req.header('If-None-Match') - log.debug "Feed: #{feed_id} " + if etag then " / #{etag}" else "" - collection.findOne _id: feed_id, (err, docs) -> - if not err and etag and docs and docs?.color == etag - log.debug "Cached: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else "" - res.sendStatus 304 - else if not err and docs and docs.data - log.debug "Req: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else "" - res.header 'etag', docs.color - body = new Buffer(docs.data, 'base64') - res.set("Content-Type", "image/png") - res.status(200).send body - else - log.debug "Redirect: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else "" - if ENV_DEV - res.redirect '/media/img/icons/circular/world.png' + log.debug "Connected to #{db?.serverConfig.s.seedlist[0].host}:#{db?.serverConfig.s.seedlist[0].port}" + if err + log.debug " ***> Error connecting: #{err}" + + app.get /\/rss_feeds\/icon\/(\d+)\/?/, (req, res) => + feed_id = parseInt(req.params[0], 10) + etag = req.header('If-None-Match') + log.debug "Feed: #{feed_id} " + if etag then " / #{etag}" else "" + collection.findOne _id: feed_id, (err, docs) -> + if not err and etag and docs and docs?.color == etag + log.debug "Cached: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else "" + res.sendStatus 304 + else if not err and docs and docs.data + log.debug "Req: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else "" + res.header 'etag', docs.color + body = new Buffer(docs.data, 'base64') + res.set("Content-Type", "image/png") + res.status(200).send body else - res.redirect 'https://www.newsblur.com/media/img/icons/circular/world.png' + log.debug "Redirect: #{feed_id}, etag: #{etag}/#{docs?.color} " + if err then "(err: #{err})" else "" + if ENV_DEV + res.redirect '/media/img/icons/circular/world.png' + else + res.redirect 'https://www.newsblur.com/media/img/icons/circular/world.png' - app.listen 3030 +exports.favicons = favicons diff --git a/node/favicons.js b/node/favicons.js index 594ee3259..2c5d30a26 100644 --- a/node/favicons.js +++ b/node/favicons.js @@ -1,97 +1,88 @@ // Generated by CoffeeScript 2.5.1 (function() { - var ENV_DEV, ENV_DOCKER, ENV_PROD, MONGODB_PORT, MONGODB_SERVER, app, log, mongo, server, url; - - app = require('express')(); - - server = require('http').Server(app); + var favicons, log, mongo; mongo = require('mongodb'); log = require('./log.js'); - ENV_DEV = process.env.NODE_ENV === 'development'; - - ENV_PROD = process.env.NODE_ENV === 'production'; - - ENV_DOCKER = process.env.NODE_ENV === 'docker'; - - MONGODB_SERVER = "db_mongo"; - - if (ENV_DEV) { - MONGODB_SERVER = 'localhost'; - } else if (ENV_PROD) { - MONGODB_SERVER = 'db-mongo.service.nyc1.consul'; - } - - MONGODB_PORT = parseInt(process.env.MONGODB_PORT || 27017, 10); - - log.debug("Starting NewsBlur Favicon server..."); - - if (!ENV_DEV && !process.env.NODE_ENV) { - log.debug("Specify NODE_ENV="); - return; - } else if (ENV_DEV) { - log.debug("Running as development server"); - } else if (ENV_DOCKER) { - log.debug("Running as docker server"); - } else { - log.debug("Running as production server"); - } - - if (ENV_DEV || ENV_DOCKER) { - url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur`; - } else { - url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred`; - } - - (async function() { - var client, collection, db, err; - try { - client = mongo.MongoClient(url, { - useUnifiedTopology: true - }); - await client.connect(); - } catch (error) { - err = error; - log.debug(`Error connecting to Mongo: ${err}`); + favicons = (app) => { + var ENV_DEV, ENV_DOCKER, ENV_PROD, MONGODB_PORT, MONGODB_SERVER, url; + ENV_DEV = process.env.NODE_ENV === 'development'; + ENV_PROD = process.env.NODE_ENV === 'production'; + ENV_DOCKER = process.env.NODE_ENV === 'docker'; + MONGODB_SERVER = "db_mongo"; + if (ENV_DEV) { + MONGODB_SERVER = 'localhost'; + } else if (ENV_PROD) { + MONGODB_SERVER = 'db-mongo.service.nyc1.consul'; + } + MONGODB_PORT = parseInt(process.env.MONGODB_PORT || 27017, 10); + log.debug("Starting NewsBlur Favicon server..."); + if (!ENV_DEV && !process.env.NODE_ENV) { + log.debug("Specify NODE_ENV="); return; + } else if (ENV_DEV) { + log.debug("Running as development server"); + } else if (ENV_DOCKER) { + log.debug("Running as docker server"); + } else { + log.debug("Running as production server"); } - db = client.db("newsblur"); - collection = db.collection("feed_icons"); - log.debug(`Connected to ${db != null ? db.serverConfig.s.seedlist[0].host : void 0}:${db != null ? db.serverConfig.s.seedlist[0].port : void 0}`); - if (err) { - log.debug(` ***> Error connecting: ${err}`); + if (ENV_DEV || ENV_DOCKER) { + url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur`; + } else { + url = `mongodb://${MONGODB_SERVER}:${MONGODB_PORT}/newsblur?replicaSet=nbset&readPreference=secondaryPreferred`; } - app.get(/\/rss_feeds\/icon\/(\d+)\/?/, (req, res) => { - var etag, feed_id; - feed_id = parseInt(req.params[0], 10); - etag = req.header('If-None-Match'); - log.debug(`Feed: ${feed_id} ` + (etag ? ` / ${etag}` : "")); - return collection.findOne({ - _id: feed_id - }, function(err, docs) { - var body; - if (!err && etag && docs && (docs != null ? docs.color : void 0) === etag) { - log.debug(`Cached: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : "")); - return res.sendStatus(304); - } else if (!err && docs && docs.data) { - log.debug(`Req: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : "")); - res.header('etag', docs.color); - body = new Buffer(docs.data, 'base64'); - res.set("Content-Type", "image/png"); - return res.status(200).send(body); - } else { - log.debug(`Redirect: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : "")); - if (ENV_DEV) { - return res.redirect('/media/img/icons/circular/world.png'); + return (async function() { + var client, collection, db, err; + try { + client = mongo.MongoClient(url, { + useUnifiedTopology: true + }); + await client.connect(); + } catch (error) { + err = error; + log.debug(`Error connecting to Mongo: ${err}`); + return; + } + db = client.db("newsblur"); + collection = db.collection("feed_icons"); + log.debug(`Connected to ${db != null ? db.serverConfig.s.seedlist[0].host : void 0}:${db != null ? db.serverConfig.s.seedlist[0].port : void 0}`); + if (err) { + log.debug(` ***> Error connecting: ${err}`); + } + return app.get(/\/rss_feeds\/icon\/(\d+)\/?/, (req, res) => { + var etag, feed_id; + feed_id = parseInt(req.params[0], 10); + etag = req.header('If-None-Match'); + log.debug(`Feed: ${feed_id} ` + (etag ? ` / ${etag}` : "")); + return collection.findOne({ + _id: feed_id + }, function(err, docs) { + var body; + if (!err && etag && docs && (docs != null ? docs.color : void 0) === etag) { + log.debug(`Cached: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : "")); + return res.sendStatus(304); + } else if (!err && docs && docs.data) { + log.debug(`Req: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : "")); + res.header('etag', docs.color); + body = new Buffer(docs.data, 'base64'); + res.set("Content-Type", "image/png"); + return res.status(200).send(body); } else { - return res.redirect('https://www.newsblur.com/media/img/icons/circular/world.png'); + log.debug(`Redirect: ${feed_id}, etag: ${etag}/${docs != null ? docs.color : void 0} ` + (err ? `(err: ${err})` : "")); + if (ENV_DEV) { + return res.redirect('/media/img/icons/circular/world.png'); + } else { + return res.redirect('https://www.newsblur.com/media/img/icons/circular/world.png'); + } } - } + }); }); - }); - return app.listen(3030); - })(); + })(); + }; + + exports.favicons = favicons; }).call(this); diff --git a/node/newsblur.coffee b/node/newsblur.coffee new file mode 100644 index 000000000..d1954b01f --- /dev/null +++ b/node/newsblur.coffee @@ -0,0 +1,16 @@ +app = require('express')() +server = require('http').Server(app) +log = require './log.js' + +original_page = require('./original_page.js').original_page +original_text = require('./original_text.js').original_text +favicons = require('./favicons.js').favicons +unread_counts = require('./unread_counts.js').unread_counts + +original_page(app) +original_text(app) +favicons(app) +unread_counts(server) + +log.debug "Starting NewsBlur Node Server" +app.listen(8008) diff --git a/node/newsblur.js b/node/newsblur.js new file mode 100644 index 000000000..cd4fd7854 --- /dev/null +++ b/node/newsblur.js @@ -0,0 +1,31 @@ +// Generated by CoffeeScript 2.5.1 +(function() { + var app, favicons, log, original_page, original_text, server, unread_counts; + + app = require('express')(); + + server = require('http').Server(app); + + log = require('./log.js'); + + original_page = require('./original_page.js').original_page; + + original_text = require('./original_text.js').original_text; + + favicons = require('./favicons.js').favicons; + + unread_counts = require('./unread_counts.js').unread_counts; + + original_page(app); + + original_text(app); + + favicons(app); + + unread_counts(server); + + log.debug("Starting NewsBlur Node Server"); + + app.listen(8008); + +}).call(this); diff --git a/node/original_page.coffee b/node/original_page.coffee index 2c8c5e44a..8e049ce07 100644 --- a/node/original_page.coffee +++ b/node/original_page.coffee @@ -1,69 +1,70 @@ -app = require('express')() -server = require('http').Server(app) path = require 'path' busboy = require 'connect-busboy' fs = require 'fs' mkdirp = require 'mkdirp' +log = require './log.js' -DEV = process.env.NODE_ENV == 'development' +original_page = (app) => + DEV = process.env.NODE_ENV == 'development' -DB_PATH = if DEV then 'originals' else '/srv/originals' + DB_PATH = if DEV then 'originals' else '/srv/originals' -server.listen 3060 + app.use busboy() -app.use busboy() + app.get /^\/original_page\/(\d+)\/?/, (req, res) => + if req.query.test + return res.end "OK" -app.get /^\/original_page\/(\d+)\/?/, (req, res) => - if req.query.test - return res.end "OK" - - feedId = parseInt(req.params[0], 10) - etag = req.header('If-None-Match') - lastModified = req.header('If-Modified-Since') - feedIdDir = splitFeedId feedId - filePath = "#{DB_PATH}/#{feedIdDir}.zhtml" - - fs.exists filePath, (exists, err) -> - console.log " ---> Loading: #{feedId} (#{filePath}). " + - "#{if exists then "" else "NOT FOUND"}" - if not exists - return res.send 404 - fs.stat filePath, (err, stats) -> - if not err and etag and stats.mtime == etag - return res.send 304 - if not err and lastModified and stats.mtime == lastModified - return res.send 304 - - fs.readFile filePath, (err, content) -> - res.header 'Etag', Date.parse(stats.mtime) - res.send content - - -app.post /^\/original_page\/(\d+)\/?/, (req, res) => - feedId = parseInt(req.params[0], 10) - feedIdDir = splitFeedId feedId - req.pipe req.busboy - req.busboy.on 'file', (fieldname, file, filename) -> - # console.log "Uploading #{fieldname} / #{file} / #{filename}" + feedId = parseInt(req.params[0], 10) + etag = req.header('If-None-Match') + lastModified = req.header('If-Modified-Since') + feedIdDir = splitFeedId feedId filePath = "#{DB_PATH}/#{feedIdDir}.zhtml" - filePathDir = path.dirname filePath - mkdirp filePathDir, (err) -> - console.log err if err - fstream = fs.createWriteStream filePath - file.pipe fstream - fstream.on 'close', -> - fs.stat filePath, (err, stats) -> - console.log err if err - console.log " ---> Saving: #{feedId} (#{filePath}) #{stats.size} bytes" - res.send "OK" + + fs.exists filePath, (exists, err) -> + log.debug "Loading: #{feedId} (#{filePath}). " + + "#{if exists then "" else "NOT FOUND"}" + if not exists + return res.send 404 + fs.stat filePath, (err, stats) -> + if not err and etag and stats.mtime == etag + return res.send 304 + if not err and lastModified and stats.mtime == lastModified + return res.send 304 + + fs.readFile filePath, (err, content) -> + res.header 'Etag', Date.parse(stats.mtime) + res.send content -splitFeedId = (feedId) -> - feedId += '' - # x2 = if feedId.length > 1 then '.' + feedId[1] else '' - rgx = /(\d+)(\d{3})/ - feedId = feedId.replace rgx, '$1' + '/' + '$2' while rgx.test(feedId) - return feedId; - + app.post /^\/original_page\/(\d+)\/?/, (req, res) => + feedId = parseInt(req.params[0], 10) + feedIdDir = splitFeedId feedId + req.pipe req.busboy + req.busboy.on 'file', (fieldname, file, filename) -> + # log.debug "Uploading #{fieldname} / #{file} / #{filename}" + filePath = "#{DB_PATH}/#{feedIdDir}.zhtml" + filePathDir = path.dirname filePath + mkdirp filePathDir, (err) -> + log.debug err if err + fstream = fs.createWriteStream filePath + file.pipe fstream + fstream.on 'close', -> + fs.stat filePath, (err, stats) -> + log.debug err if err + log.debug "Saving: #{feedId} (#{filePath}) #{stats.size} bytes" + res.send "OK" -console.log " ---> Starting Original Page server #{if DEV then "on DEV" else "in production"}" + + splitFeedId = (feedId) -> + feedId += '' + # x2 = if feedId.length > 1 then '.' + feedId[1] else '' + rgx = /(\d+)(\d{3})/ + feedId = feedId.replace rgx, '$1' + '/' + '$2' while rgx.test(feedId) + return feedId; + + + log.debug "Starting Original Page server #{if DEV then "on DEV" else "in production"}" + + +exports.original_page = original_page diff --git a/node/original_page.js b/node/original_page.js index 90baf6c88..44713b04e 100644 --- a/node/original_page.js +++ b/node/original_page.js @@ -1,10 +1,6 @@ // Generated by CoffeeScript 2.5.1 (function() { - var DB_PATH, DEV, app, busboy, fs, mkdirp, path, server, splitFeedId; - - app = require('express')(); - - server = require('http').Server(app); + var busboy, fs, log, mkdirp, original_page, path; path = require('path'); @@ -14,85 +10,84 @@ mkdirp = require('mkdirp'); - DEV = process.env.NODE_ENV === 'development'; + log = require('./log.js'); - DB_PATH = DEV ? 'originals' : '/srv/originals'; - - server.listen(3060); - - app.use(busboy()); - - app.get(/^\/original_page\/(\d+)\/?/, (req, res) => { - var etag, feedId, feedIdDir, filePath, lastModified; - if (req.query.test) { - return res.end("OK"); - } - feedId = parseInt(req.params[0], 10); - etag = req.header('If-None-Match'); - lastModified = req.header('If-Modified-Since'); - feedIdDir = splitFeedId(feedId); - filePath = `${DB_PATH}/${feedIdDir}.zhtml`; - return fs.exists(filePath, function(exists, err) { - console.log(` ---> Loading: ${feedId} (${filePath}). ` + `${exists ? "" : "NOT FOUND"}`); - if (!exists) { - return res.send(404); + original_page = (app) => { + var DB_PATH, DEV, splitFeedId; + DEV = process.env.NODE_ENV === 'development'; + DB_PATH = DEV ? 'originals' : '/srv/originals'; + app.use(busboy()); + app.get(/^\/original_page\/(\d+)\/?/, (req, res) => { + var etag, feedId, feedIdDir, filePath, lastModified; + if (req.query.test) { + return res.end("OK"); } - return fs.stat(filePath, function(err, stats) { - if (!err && etag && stats.mtime === etag) { - return res.send(304); - } - if (!err && lastModified && stats.mtime === lastModified) { - return res.send(304); - } - return fs.readFile(filePath, function(err, content) { - res.header('Etag', Date.parse(stats.mtime)); - return res.send(content); - }); - }); - }); - }); - - app.post(/^\/original_page\/(\d+)\/?/, (req, res) => { - var feedId, feedIdDir; - feedId = parseInt(req.params[0], 10); - feedIdDir = splitFeedId(feedId); - req.pipe(req.busboy); - return req.busboy.on('file', function(fieldname, file, filename) { - var filePath, filePathDir; - // console.log "Uploading #{fieldname} / #{file} / #{filename}" + feedId = parseInt(req.params[0], 10); + etag = req.header('If-None-Match'); + lastModified = req.header('If-Modified-Since'); + feedIdDir = splitFeedId(feedId); filePath = `${DB_PATH}/${feedIdDir}.zhtml`; - filePathDir = path.dirname(filePath); - return mkdirp(filePathDir, function(err) { - var fstream; - if (err) { - console.log(err); + return fs.exists(filePath, function(exists, err) { + log.debug(`Loading: ${feedId} (${filePath}). ` + `${exists ? "" : "NOT FOUND"}`); + if (!exists) { + return res.send(404); } - fstream = fs.createWriteStream(filePath); - file.pipe(fstream); - return fstream.on('close', function() { - return fs.stat(filePath, function(err, stats) { - if (err) { - console.log(err); - } - console.log(` ---> Saving: ${feedId} (${filePath}) ${stats.size} bytes`); - return res.send("OK"); + return fs.stat(filePath, function(err, stats) { + if (!err && etag && stats.mtime === etag) { + return res.send(304); + } + if (!err && lastModified && stats.mtime === lastModified) { + return res.send(304); + } + return fs.readFile(filePath, function(err, content) { + res.header('Etag', Date.parse(stats.mtime)); + return res.send(content); }); }); }); }); - }); - - splitFeedId = function(feedId) { - var rgx; - feedId += ''; - // x2 = if feedId.length > 1 then '.' + feedId[1] else '' - rgx = /(\d+)(\d{3})/; - while (rgx.test(feedId)) { - feedId = feedId.replace(rgx, '$1' + '/' + '$2'); - } - return feedId; + app.post(/^\/original_page\/(\d+)\/?/, (req, res) => { + var feedId, feedIdDir; + feedId = parseInt(req.params[0], 10); + feedIdDir = splitFeedId(feedId); + req.pipe(req.busboy); + return req.busboy.on('file', function(fieldname, file, filename) { + var filePath, filePathDir; + // log.debug "Uploading #{fieldname} / #{file} / #{filename}" + filePath = `${DB_PATH}/${feedIdDir}.zhtml`; + filePathDir = path.dirname(filePath); + return mkdirp(filePathDir, function(err) { + var fstream; + if (err) { + log.debug(err); + } + fstream = fs.createWriteStream(filePath); + file.pipe(fstream); + return fstream.on('close', function() { + return fs.stat(filePath, function(err, stats) { + if (err) { + log.debug(err); + } + log.debug(`Saving: ${feedId} (${filePath}) ${stats.size} bytes`); + return res.send("OK"); + }); + }); + }); + }); + }); + splitFeedId = function(feedId) { + var rgx; + feedId += ''; + // x2 = if feedId.length > 1 then '.' + feedId[1] else '' + rgx = /(\d+)(\d{3})/; + while (rgx.test(feedId)) { + feedId = feedId.replace(rgx, '$1' + '/' + '$2'); + } + return feedId; + }; + return log.debug(`Starting Original Page server ${DEV ? "on DEV" : "in production"}`); }; - console.log(` ---> Starting Original Page server ${DEV ? "on DEV" : "in production"}`); + exports.original_page = original_page; }).call(this); diff --git a/node/original_text.coffee b/node/original_text.coffee index 23ce33be3..7ee734793 100644 --- a/node/original_text.coffee +++ b/node/original_text.coffee @@ -1,36 +1,35 @@ Mercury = require('@postlight/mercury-parser') -app = require('express')() -server = require('http').Server(app) log = require './log.js' -DEV = process.env.NODE_ENV == 'development' +original_text = (app) => + DEV = process.env.NODE_ENV == 'development' -log.debug "Starting NewsBlur Original Text Fetcher / Mercury Parser..." -if !DEV and !process.env.NODE_ENV - log.debug "Specify NODE_ENV=" - return -else if DEV - log.debug "Running as development server" -else - log.debug "Running as production server" + log.debug "Starting NewsBlur Original Text Fetcher / Mercury Parser..." + if !DEV and !process.env.NODE_ENV + log.debug "Specify NODE_ENV=" + return + else if DEV + log.debug "Running as development server" + else + log.debug "Running as production server" -app.get /\/rss_feeds\/original_text_fetcher\/?/, (req, res) => - res.setHeader 'Content-Type', 'application/json' - if req.query.test - return res.end "OK" - - url = req.query.url - if !url - log.debug "Missing url" - return res.end JSON.stringify error: "Missing `url` query parameter." - - api_key = req.header('x-api-key') or req.query.apikey - if not DEV and (!api_key or api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") == -1) - log.debug "Mismatched API key: #{url} / #{api_key}" - return res.end JSON.stringify error: "Invalid API key. You need to set up your own Original Text server." + app.get /\/rss_feeds\/original_text_fetcher\/?/, (req, res) => + res.setHeader 'Content-Type', 'application/json' + if req.query.test + return res.end "OK" - Mercury.parse(url).then (result) => - log.debug "Fetched: #{url}" - res.end JSON.stringify result + url = req.query.url + if !url + log.debug "Missing url" + return res.end JSON.stringify error: "Missing `url` query parameter." + + api_key = req.header('x-api-key') or req.query.apikey + if not DEV and (!api_key or api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") == -1) + log.debug "Mismatched API key: #{url} / #{api_key}" + return res.end JSON.stringify error: "Invalid API key. You need to set up your own Original Text server." + + Mercury.parse(url).then (result) => + log.debug "Fetched: #{url}" + res.end JSON.stringify result -app.listen 4040 +exports.original_text = original_text diff --git a/node/original_text.js b/node/original_text.js index 68dc29a01..4923a4755 100644 --- a/node/original_text.js +++ b/node/original_text.js @@ -1,54 +1,50 @@ // Generated by CoffeeScript 2.5.1 (function() { - var DEV, Mercury, app, log, server; + var Mercury, log, original_text; Mercury = require('@postlight/mercury-parser'); - app = require('express')(); - - server = require('http').Server(app); - log = require('./log.js'); - DEV = process.env.NODE_ENV === 'development'; - - log.debug("Starting NewsBlur Original Text Fetcher / Mercury Parser..."); - - if (!DEV && !process.env.NODE_ENV) { - log.debug("Specify NODE_ENV="); - return; - } else if (DEV) { - log.debug("Running as development server"); - } else { - log.debug("Running as production server"); - } - - app.get(/\/rss_feeds\/original_text_fetcher\/?/, (req, res) => { - var api_key, url; - res.setHeader('Content-Type', 'application/json'); - if (req.query.test) { - return res.end("OK"); + original_text = (app) => { + var DEV; + DEV = process.env.NODE_ENV === 'development'; + log.debug("Starting NewsBlur Original Text Fetcher / Mercury Parser..."); + if (!DEV && !process.env.NODE_ENV) { + log.debug("Specify NODE_ENV="); + return; + } else if (DEV) { + log.debug("Running as development server"); + } else { + log.debug("Running as production server"); } - url = req.query.url; - if (!url) { - log.debug("Missing url"); - return res.end(JSON.stringify({ - error: "Missing `url` query parameter." - })); - } - api_key = req.header('x-api-key') || req.query.apikey; - if (!DEV && (!api_key || api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") === -1)) { - log.debug(`Mismatched API key: ${url} / ${api_key}`); - return res.end(JSON.stringify({ - error: "Invalid API key. You need to set up your own Original Text server." - })); - } - return Mercury.parse(url).then((result) => { - log.debug(`Fetched: ${url}`); - return res.end(JSON.stringify(result)); + return app.get(/\/rss_feeds\/original_text_fetcher\/?/, (req, res) => { + var api_key, url; + res.setHeader('Content-Type', 'application/json'); + if (req.query.test) { + return res.end("OK"); + } + url = req.query.url; + if (!url) { + log.debug("Missing url"); + return res.end(JSON.stringify({ + error: "Missing `url` query parameter." + })); + } + api_key = req.header('x-api-key') || req.query.apikey; + if (!DEV && (!api_key || api_key.indexOf("djtXZrSIEfDa3Dex9FQ9AR") === -1)) { + log.debug(`Mismatched API key: ${url} / ${api_key}`); + return res.end(JSON.stringify({ + error: "Invalid API key. You need to set up your own Original Text server." + })); + } + return Mercury.parse(url).then((result) => { + log.debug(`Fetched: ${url}`); + return res.end(JSON.stringify(result)); + }); }); - }); + }; - app.listen(4040); + exports.original_text = original_text; }).call(this); diff --git a/node/unread_counts.coffee b/node/unread_counts.coffee index 9eb532351..e0499eb07 100644 --- a/node/unread_counts.coffee +++ b/node/unread_counts.coffee @@ -2,103 +2,87 @@ fs = require 'fs' redis = require 'redis' log = require './log.js' -ENV_DEV = process.env.NODE_ENV == 'development' -ENV_PROD = process.env.NODE_ENV == 'production' -ENV_DOCKER = process.env.NODE_ENV == 'docker' -REDIS_SERVER = "redis" -if ENV_DEV - REDIS_SERVER = 'localhost' -else if ENV_PROD - REDIS_SERVER = 'db-redis-user.service.nyc1.consul' -SECURE = !!process.env.NODE_SSL -REDIS_PORT = if ENV_DOCKER then 6579 else 6379 +unread_counts = (server) => + ENV_DEV = process.env.NODE_ENV == 'development' + ENV_PROD = process.env.NODE_ENV == 'production' + ENV_DOCKER = process.env.NODE_ENV == 'docker' + REDIS_SERVER = "redis" + if ENV_DEV + REDIS_SERVER = 'localhost' + else if ENV_PROD + REDIS_SERVER = 'db-redis-user.service.nyc1.consul' + SECURE = !!process.env.NODE_SSL + REDIS_PORT = if ENV_DOCKER then 6579 else 6379 -# client = redis.createClient 6379, REDIS_SERVER + # client = redis.createClient 6379, REDIS_SERVER -# RedisStore = require 'socket.io/lib/stores/redis' -# rpub = redis.createClient 6379, REDIS_SERVER -# rsub = redis.createClient 6379, REDIS_SERVER -# rclient = redis.createClient 6379, REDIS_SERVER + # RedisStore = require 'socket.io/lib/stores/redis' + # rpub = redis.createClient 6379, REDIS_SERVER + # rsub = redis.createClient 6379, REDIS_SERVER + # rclient = redis.createClient 6379, REDIS_SERVER -log.debug "Starting NewsBlur unread count server..." -if !ENV_DEV and !process.env.NODE_ENV - log.debug "Specify NODE_ENV=" - return -else if ENV_DEV - log.debug "Running as development server" -else - log.debug "Running as production server" - - -if SECURE - privateKey = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.key').toString() - certificate = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.crt').toString() - # ca = fs.readFileSync('./config/certificates/intermediate.crt').toString() - options = - port: 8889 - key: privateKey - cert: certificate - app = require('https').createServer options - io = require('socket.io')(app, path: "/v3/socket.io") - app.listen options.port - log.debug "Listening securely on port #{options.port}" -else - options = - port: 8888 - app = require('http').createServer() - io = require('socket.io')(app, path: "/v3/socket.io") - app.listen options.port - log.debug "Listening on port #{options.port}" - -# io.set('transports', ['websocket']) - -# io.set 'store', new RedisStore -# redisPub : rpub -# redisSub : rsub -# redisClient : rclient - -io.on 'connection', (socket) -> - ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address - - socket.on 'subscribe:feeds', (@feeds, @username) => - log.info @username, "Connecting (#{@feeds.length} feeds, #{ip})," + - " (#{io.engine.clientsCount} connected) " + - " #{if SECURE then "(SSL)" else "(non-SSL)"}" + log.debug "Starting NewsBlur unread count server..." + if !ENV_DEV and !process.env.NODE_ENV + log.debug "Specify NODE_ENV=" + return + else if ENV_DEV + log.debug "Running as development server" + else + log.debug "Running as production server" - if not @username - return - - socket.on "error", (err) -> - log.debug "Error (socket): #{err}" - socket.subscribe?.quit() - socket.subscribe = redis.createClient REDIS_PORT, REDIS_SERVER - socket.subscribe.on "error", (err) => - log.info @username, "Error: #{err} (#{@feeds.length} feeds)" - socket.subscribe?.quit() - socket.subscribe.on "connect", => - log.info @username, "Connected (#{@feeds.length} feeds, #{ip})," + - " (#{io.engine.clientsCount} connected) " + - " #{if SECURE then "(SSL)" else "(non-SSL)"}" - socket.subscribe.subscribe @feeds - feeds_story = @feeds.map (f) -> "#{f}:story" - socket.subscribe.subscribe feeds_story - socket.subscribe.subscribe @username + io = require('socket.io')(server, path: "/v3/socket.io") - socket.subscribe.on 'message', (channel, message) => - event_name = 'feed:update' - if channel == @username - event_name = 'user:update' - else if channel.indexOf(':story') >= 0 - event_name = 'feed:story:new' - log.info @username, "Update on #{channel}: #{event_name} - #{message}" - socket.emit event_name, channel, message + # io.set('transports', ['websocket']) - socket.on 'disconnect', () => - socket.subscribe?.quit() - log.info @username, "Disconnect (#{@feeds?.length} feeds, #{ip})," + - " there are now #{io.engine.clientsCount} users. " + + # io.set 'store', new RedisStore + # redisPub : rpub + # redisSub : rsub + # redisClient : rclient + + io.on 'connection', (socket) -> + ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address + + socket.on 'subscribe:feeds', (@feeds, @username) => + log.info @username, "Connecting (#{@feeds.length} feeds, #{ip})," + + " (#{io.engine.clientsCount} connected) " + " #{if SECURE then "(SSL)" else "(non-SSL)"}" + + if not @username + return + + socket.on "error", (err) -> + log.debug "Error (socket): #{err}" + socket.subscribe?.quit() + socket.subscribe = redis.createClient REDIS_PORT, REDIS_SERVER + socket.subscribe.on "error", (err) => + log.info @username, "Error: #{err} (#{@feeds.length} feeds)" + socket.subscribe?.quit() + socket.subscribe.on "connect", => + log.info @username, "Connected (#{@feeds.length} feeds, #{ip})," + + " (#{io.engine.clientsCount} connected) " + + " #{if SECURE then "(SSL)" else "(non-SSL)"}" + socket.subscribe.subscribe @feeds + feeds_story = @feeds.map (f) -> "#{f}:story" + socket.subscribe.subscribe feeds_story + socket.subscribe.subscribe @username -io.sockets.on 'error', (err) -> - log.debug "Error (sockets): #{err}" + socket.subscribe.on 'message', (channel, message) => + event_name = 'feed:update' + if channel == @username + event_name = 'user:update' + else if channel.indexOf(':story') >= 0 + event_name = 'feed:story:new' + log.info @username, "Update on #{channel}: #{event_name} - #{message}" + socket.emit event_name, channel, message + + socket.on 'disconnect', () => + socket.subscribe?.quit() + log.info @username, "Disconnect (#{@feeds?.length} feeds, #{ip})," + + " there are now #{io.engine.clientsCount} users. " + + " #{if SECURE then "(SSL)" else "(non-SSL)"}" + + io.sockets.on 'error', (err) -> + log.debug "Error (sockets): #{err}" + +exports.unread_counts = unread_counts diff --git a/node/unread_counts.js b/node/unread_counts.js index e38986fd5..252eace2c 100644 --- a/node/unread_counts.js +++ b/node/unread_counts.js @@ -1,6 +1,6 @@ // Generated by CoffeeScript 2.5.1 (function() { - var ENV_DEV, ENV_DOCKER, ENV_PROD, REDIS_PORT, REDIS_SERVER, SECURE, app, certificate, fs, io, log, options, privateKey, redis; + var fs, log, redis, unread_counts; fs = require('fs'); @@ -8,130 +8,101 @@ log = require('./log.js'); - ENV_DEV = process.env.NODE_ENV === 'development'; + unread_counts = (server) => { + var ENV_DEV, ENV_DOCKER, ENV_PROD, REDIS_PORT, REDIS_SERVER, SECURE, io; + ENV_DEV = process.env.NODE_ENV === 'development'; + ENV_PROD = process.env.NODE_ENV === 'production'; + ENV_DOCKER = process.env.NODE_ENV === 'docker'; + REDIS_SERVER = "redis"; + if (ENV_DEV) { + REDIS_SERVER = 'localhost'; + } else if (ENV_PROD) { + REDIS_SERVER = 'db-redis-user.service.nyc1.consul'; + } + SECURE = !!process.env.NODE_SSL; + REDIS_PORT = ENV_DOCKER ? 6579 : 6379; + // client = redis.createClient 6379, REDIS_SERVER - ENV_PROD = process.env.NODE_ENV === 'production'; - - ENV_DOCKER = process.env.NODE_ENV === 'docker'; - - REDIS_SERVER = "redis"; - - if (ENV_DEV) { - REDIS_SERVER = 'localhost'; - } else if (ENV_PROD) { - REDIS_SERVER = 'db-redis-user.service.nyc1.consul'; - } - - SECURE = !!process.env.NODE_SSL; - - REDIS_PORT = ENV_DOCKER ? 6579 : 6379; - - // client = redis.createClient 6379, REDIS_SERVER - - // RedisStore = require 'socket.io/lib/stores/redis' - // rpub = redis.createClient 6379, REDIS_SERVER - // rsub = redis.createClient 6379, REDIS_SERVER - // rclient = redis.createClient 6379, REDIS_SERVER - log.debug("Starting NewsBlur unread count server..."); - - if (!ENV_DEV && !process.env.NODE_ENV) { - log.debug("Specify NODE_ENV="); - return; - } else if (ENV_DEV) { - log.debug("Running as development server"); - } else { - log.debug("Running as production server"); - } - - if (SECURE) { - privateKey = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.key').toString(); - certificate = fs.readFileSync('/srv/newsblur/config/certificates/newsblur.com.crt').toString(); - // ca = fs.readFileSync('./config/certificates/intermediate.crt').toString() - options = { - port: 8889, - key: privateKey, - cert: certificate - }; - app = require('https').createServer(options); - io = require('socket.io')(app, { + // RedisStore = require 'socket.io/lib/stores/redis' + // rpub = redis.createClient 6379, REDIS_SERVER + // rsub = redis.createClient 6379, REDIS_SERVER + // rclient = redis.createClient 6379, REDIS_SERVER + log.debug("Starting NewsBlur unread count server..."); + if (!ENV_DEV && !process.env.NODE_ENV) { + log.debug("Specify NODE_ENV="); + return; + } else if (ENV_DEV) { + log.debug("Running as development server"); + } else { + log.debug("Running as production server"); + } + io = require('socket.io')(server, { path: "/v3/socket.io" }); - app.listen(options.port); - log.debug(`Listening securely on port ${options.port}`); - } else { - options = { - port: 8888 - }; - app = require('http').createServer(); - io = require('socket.io')(app, { - path: "/v3/socket.io" - }); - app.listen(options.port); - log.debug(`Listening on port ${options.port}`); - } + // io.set('transports', ['websocket']) - // io.set('transports', ['websocket']) - - // io.set 'store', new RedisStore - // redisPub : rpub - // redisSub : rsub - // redisClient : rclient - io.on('connection', function(socket) { - var ip; - ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address; - socket.on('subscribe:feeds', (feeds, username) => { - var ref; - this.feeds = feeds; - this.username = username; - log.info(this.username, `Connecting (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`); - if (!this.username) { - return; - } - socket.on("error", function(err) { - return log.debug(`Error (socket): ${err}`); - }); - if ((ref = socket.subscribe) != null) { - ref.quit(); - } - socket.subscribe = redis.createClient(REDIS_PORT, REDIS_SERVER); - socket.subscribe.on("error", (err) => { - var ref1; - log.info(this.username, `Error: ${err} (${this.feeds.length} feeds)`); - return (ref1 = socket.subscribe) != null ? ref1.quit() : void 0; - }); - socket.subscribe.on("connect", () => { - var feeds_story; - log.info(this.username, `Connected (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`); - socket.subscribe.subscribe(this.feeds); - feeds_story = this.feeds.map(function(f) { - return `${f}:story`; - }); - socket.subscribe.subscribe(feeds_story); - return socket.subscribe.subscribe(this.username); - }); - return socket.subscribe.on('message', (channel, message) => { - var event_name; - event_name = 'feed:update'; - if (channel === this.username) { - event_name = 'user:update'; - } else if (channel.indexOf(':story') >= 0) { - event_name = 'feed:story:new'; + // io.set 'store', new RedisStore + // redisPub : rpub + // redisSub : rsub + // redisClient : rclient + io.on('connection', function(socket) { + var ip; + ip = socket.handshake.headers['X-Forwarded-For'] || socket.handshake.address; + socket.on('subscribe:feeds', (feeds, username) => { + var ref; + this.feeds = feeds; + this.username = username; + log.info(this.username, `Connecting (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`); + if (!this.username) { + return; } - log.info(this.username, `Update on ${channel}: ${event_name} - ${message}`); - return socket.emit(event_name, channel, message); + socket.on("error", function(err) { + return log.debug(`Error (socket): ${err}`); + }); + if ((ref = socket.subscribe) != null) { + ref.quit(); + } + socket.subscribe = redis.createClient(REDIS_PORT, REDIS_SERVER); + socket.subscribe.on("error", (err) => { + var ref1; + log.info(this.username, `Error: ${err} (${this.feeds.length} feeds)`); + return (ref1 = socket.subscribe) != null ? ref1.quit() : void 0; + }); + socket.subscribe.on("connect", () => { + var feeds_story; + log.info(this.username, `Connected (${this.feeds.length} feeds, ${ip}),` + ` (${io.engine.clientsCount} connected) ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`); + socket.subscribe.subscribe(this.feeds); + feeds_story = this.feeds.map(function(f) { + return `${f}:story`; + }); + socket.subscribe.subscribe(feeds_story); + return socket.subscribe.subscribe(this.username); + }); + return socket.subscribe.on('message', (channel, message) => { + var event_name; + event_name = 'feed:update'; + if (channel === this.username) { + event_name = 'user:update'; + } else if (channel.indexOf(':story') >= 0) { + event_name = 'feed:story:new'; + } + log.info(this.username, `Update on ${channel}: ${event_name} - ${message}`); + return socket.emit(event_name, channel, message); + }); + }); + return socket.on('disconnect', () => { + var ref, ref1; + if ((ref = socket.subscribe) != null) { + ref.quit(); + } + return log.info(this.username, `Disconnect (${(ref1 = this.feeds) != null ? ref1.length : void 0} feeds, ${ip}),` + ` there are now ${io.engine.clientsCount} users. ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`); }); }); - return socket.on('disconnect', () => { - var ref, ref1; - if ((ref = socket.subscribe) != null) { - ref.quit(); - } - return log.info(this.username, `Disconnect (${(ref1 = this.feeds) != null ? ref1.length : void 0} feeds, ${ip}),` + ` there are now ${io.engine.clientsCount} users. ` + ` ${SECURE ? "(SSL)" : "(non-SSL)"}`); + return io.sockets.on('error', function(err) { + return log.debug(`Error (sockets): ${err}`); }); - }); + }; - io.sockets.on('error', function(err) { - return log.debug(`Error (sockets): ${err}`); - }); + exports.unread_counts = unread_counts; }).call(this); diff --git a/templates/analyzer/popularity_query.xhtml b/templates/analyzer/popularity_query.xhtml index b6f5bad2c..34bc64dab 100644 --- a/templates/analyzer/popularity_query.xhtml +++ b/templates/analyzer/popularity_query.xhtml @@ -12,12 +12,12 @@
-

NewsBlur Publisher Keyword Popularity XLSX Creator for YC Founders

+

NewsBlur News Popularity Spreadsheet Creator

Search for topics across millions of stories from millions of publishers
{% if success %} -

Got it!
Email should be sent within the hour.

+

Got it!
Email should be sent within the next few minutes.

{% else %}
{% csrf_token %}
@@ -43,4 +43,4 @@
-{% endblock %} \ No newline at end of file +{% endblock %}