Rewriting original page server to use async.

.gitignore

@@ -58,6 +58,7 @@ docker/postgres/postgres.conf
 
 # Local configuration file (sdk path, etc)
 /originals
+/node/originals
 media/safari/NewsBlur.safariextz
 
 # IDE files

ansible/roles/redis-exporter/tasks/main.yml

@@ -18,7 +18,7 @@
       redis_target: "redis-user"
       redis_port: 6381
     - port: 9122
-      redis_target: "redis-sessions"
+      redis_target: "redis-session"
       redis_port: 6382
     - port: 9123
       redis_target: "redis-story"
@@ -39,7 +39,7 @@
     - port: 9121
       redis_target: "redis-user"
     - port: 9122
-      redis_target: "redis-sessions"
+      redis_target: "redis-session"
     - port: 9123
       redis_target: "redis-story"
     - port: 9124

apps/reader/views.py

@@ -880,7 +880,6 @@ def load_feed_page(request, feed_id):
         raise Http404
     
     feed = Feed.get_by_id(feed_id)
-    
     if feed and feed.has_page and not feed.has_page_exception:
         if settings.BACKED_BY_AWS.get('pages_on_node'):
             domain = Site.objects.get_current().domain
@@ -889,8 +888,9 @@ def load_feed_page(request, feed_id):
                 feed.pk,
             )
             try:
-                page_response = requests.get(url)
+                page_response = requests.get(url, verify=not settings.DEBUG)
-            except requests.ConnectionError:
+            except requests.ConnectionError as e:
+                logging.user(request, f"~FR~SBError loading original page: {url} {e}")
                 page_response = None
             if page_response and page_response.status_code == 200:
                 response = HttpResponse(page_response.content, content_type="text/html; charset=utf-8")

docker-compose.yml

@@ -50,6 +50,7 @@ services:
       - 8008:8008
     volumes: 
       - ${PWD}/node:/srv
+      - ${PWD}/node/originals:/srv/originals
   
   imageproxy:
     container_name: imageproxy

node/original_page.coffee

@@ -4,56 +4,66 @@ fs = require 'fs'
 mkdirp = require 'mkdirp'
 log    = require './log.js'
 
+fsPromises = fs.promises
+mkdirpPromise = require('util').promisify(mkdirp)
+
 original_page = (app) =>
     DEV = process.env.NODE_ENV == 'development' || process.env.NODE_ENV == 'docker' || process.env.NODE_ENV == 'debug'
 
-    DB_PATH = if DEV then 'originals' else '/srv/originals'
+    DB_PATH = '/srv/originals'
 
     app.use busboy()
 
-    app.get /^\/original_page\/(\d+)\/?/, (req, res) =>
+    app.get /^\/original_page\/(\d+)\/?/, (req, res) ->
-        if req.query.test
+        return res.end "OK" if req.query.test
-            return res.end "OK"
 
-        feedId = parseInt(req.params[0], 10)
+        feedId = parseInt req.params[0], 10
-        etag = req.header('If-None-Match')
+        etag = req.header 'If-None-Match'
-        lastModified = req.header('If-Modified-Since')
+        lastModified = req.header 'If-Modified-Since'
         feedIdDir = splitFeedId feedId
         filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
 
-        fs.exists filePath, (exists, err) ->
+        # Convert to async flow with try/catch using CoffeeScript's then/catch for Promises
-            log.debug "Loading: #{feedId} (#{filePath}). " +
+        fsPromises.stat(filePath).then (stats) ->
-                        "#{if exists then "" else "NOT FOUND"}"
+            fileEtag = Date.parse(stats.mtime).toString()
-            if not exists
+            if etag is fileEtag or lastModified is stats.mtime.toISOString()
-                return res.sendStatus 404
+                log.debug "Not modified: #{feedId} (#{filePath})"
-            fs.stat filePath, (err, stats) ->
+                res.sendStatus 304
-                if not err and etag and stats.mtime == etag
+            else
-                    return res.sendStatus 304
+                fsPromises.readFile(filePath).then (content) ->
-                if not err and lastModified and stats.mtime == lastModified
+                    log.debug "Sending: #{feedId} (#{filePath}) #{stats.size} bytes"
-                    return res.sendStatus 304
+                    res.header 'Etag', fileEtag
-            
-                fs.readFile filePath, (err, content) ->
-                    res.header 'Etag', Date.parse(stats.mtime)
                     res.send content
+        .catch (err) ->
+            if err.code is 'ENOENT'
+                log.debug "Original page not found: #{feedId} (#{filePath})"
+                res.sendStatus 404
+            else
+                log.debug "Error reading original page: #{feedId} (#{filePath}) #{err}"
+                res.sendStatus 500
 
 
-    app.post /^\/original_page\/(\d+)\/?/, (req, res) =>
+    app.post /^\/original_page\/(\d+)\/?/, (req, res) ->
-        feedId = parseInt(req.params[0], 10)
+        feedId = parseInt req.params[0], 10
         feedIdDir = splitFeedId feedId
-        req.pipe req.busboy
-        req.busboy.on 'file', (fieldname, file, filename) ->
-            # log.debug "Uploading #{fieldname} / #{file} / #{filename}"
         filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
         filePathDir = path.dirname filePath
-            mkdirp filePathDir, (err) ->
+
-                log.debug err if err
+        # Ensure directory exists before proceeding
+        mkdirpPromise(filePathDir).then ->
             fstream = fs.createWriteStream filePath
+            req.pipe req.busboy
+
+            req.busboy.on 'file', (fieldname, file, filename) ->
                 file.pipe fstream
+
                 fstream.on 'close', ->
-                    fs.stat filePath, (err, stats) ->
+                    fsPromises.stat(filePath).then (stats) ->
-                        log.debug err if err
                         log.debug "Saving: #{feedId} (#{filePath}) #{stats.size} bytes"
                         res.send "OK"
+        .catch (err) ->
+            log.debug err
+            res.sendStatus 500
 
 
     splitFeedId = (feedId) ->

node/original_page.js

@@ -1,6 +1,6 @@
 // Generated by CoffeeScript 2.6.1
 (function() {
-  var busboy, fs, log, mkdirp, original_page, path;
+  var busboy, fs, fsPromises, log, mkdirp, mkdirpPromise, original_page, path;
 
   path = require('path');
 
@@ -12,12 +12,16 @@
 
   log = require('./log.js');
 
+  fsPromises = fs.promises;
+
+  mkdirpPromise = require('util').promisify(mkdirp);
+
   original_page = (app) => {
     var DB_PATH, DEV, splitFeedId;
     DEV = process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'docker' || process.env.NODE_ENV === 'debug';
-    DB_PATH = DEV ? 'originals' : '/srv/originals';
+    DB_PATH = '/srv/originals';
     app.use(busboy());
-    app.get(/^\/original_page\/(\d+)\/?/, (req, res) => {
+    app.get(/^\/original_page\/(\d+)\/?/, function(req, res) {
       var etag, feedId, feedIdDir, filePath, lastModified;
       if (req.query.test) {
         return res.end("OK");
@@ -27,52 +31,53 @@
       lastModified = req.header('If-Modified-Since');
       feedIdDir = splitFeedId(feedId);
       filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
-      return fs.exists(filePath, function(exists, err) {
+      // Convert to async flow with try/catch using CoffeeScript's then/catch for Promises
-        log.debug(`Loading: ${feedId} (${filePath}). ` + `${exists ? "" : "NOT FOUND"}`);
+      return fsPromises.stat(filePath).then(function(stats) {
-        if (!exists) {
+        var fileEtag;
-          return res.sendStatus(404);
+        fileEtag = Date.parse(stats.mtime).toString();
-        }
+        if (etag === fileEtag || lastModified === stats.mtime.toISOString()) {
-        return fs.stat(filePath, function(err, stats) {
+          log.debug(`Not modified: ${feedId} (${filePath})`);
-          if (!err && etag && stats.mtime === etag) {
           return res.sendStatus(304);
-          }
+        } else {
-          if (!err && lastModified && stats.mtime === lastModified) {
+          return fsPromises.readFile(filePath).then(function(content) {
-            return res.sendStatus(304);
+            log.debug(`Sending: ${feedId} (${filePath}) ${stats.size} bytes`);
-          }
+            res.header('Etag', fileEtag);
-          return fs.readFile(filePath, function(err, content) {
-            res.header('Etag', Date.parse(stats.mtime));
             return res.send(content);
           });
+        }
+      }).catch(function(err) {
+        if (err.code === 'ENOENT') {
+          log.debug(`Original page not found: ${feedId} (${filePath})`);
+          return res.sendStatus(404);
+        } else {
+          log.debug(`Error reading original page: ${feedId} (${filePath}) ${err}`);
+          return res.sendStatus(500);
+        }
       });
     });
-    });
+    app.post(/^\/original_page\/(\d+)\/?/, function(req, res) {
-    app.post(/^\/original_page\/(\d+)\/?/, (req, res) => {
+      var feedId, feedIdDir, filePath, filePathDir;
-      var feedId, feedIdDir;
       feedId = parseInt(req.params[0], 10);
       feedIdDir = splitFeedId(feedId);
-      req.pipe(req.busboy);
-      return req.busboy.on('file', function(fieldname, file, filename) {
-        var filePath, filePathDir;
-        // log.debug "Uploading #{fieldname} / #{file} / #{filename}"
       filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
       filePathDir = path.dirname(filePath);
-        return mkdirp(filePathDir, function(err) {
+      // Ensure directory exists before proceeding
+      return mkdirpPromise(filePathDir).then(function() {
         var fstream;
-          if (err) {
-            log.debug(err);
-          }
         fstream = fs.createWriteStream(filePath);
+        req.pipe(req.busboy);
+        return req.busboy.on('file', function(fieldname, file, filename) {
           file.pipe(fstream);
           return fstream.on('close', function() {
-            return fs.stat(filePath, function(err, stats) {
+            return fsPromises.stat(filePath).then(function(stats) {
-              if (err) {
-                log.debug(err);
-              }
               log.debug(`Saving: ${feedId} (${filePath}) ${stats.size} bytes`);
               return res.send("OK");
             });
           });
         });
+      }).catch(function(err) {
+        log.debug(err);
+        return res.sendStatus(500);
       });
     });
     splitFeedId = function(feedId) {