Using S3 as a backup for node original page server.

apps/reader/views.py

@@ -597,19 +597,18 @@ def load_feed_page(request, feed_id):
                 settings.ORIGINAL_PAGE_SERVER,
                 feed.pk,
             )
-            import pdb; pdb.set_trace()
-            page_response = requests.get(url, headers={
-                'If-None-Match': request.headers,
-            })
-            response = HttpResponse(page_response.content, mimetype="text/html; charset=utf-8")
-            response['Content-Encoding'] = 'gzip'
-            response['Last-Modified'] = page_response.headers.get('Last-modified')
-            response['Etag'] = page_response.headers.get('Etag')
-            response['Content-Length'] = str(len(page_response.content))
-            logging.user(request, "~FYLoading original page, proxied from node: ~SB%s bytes" %
-                         (len(page_response.content)))
-            return response
-        elif settings.BACKED_BY_AWS['pages_on_s3'] and feed.s3_page:
+            page_response = requests.get(url)
+            if page_response.status_code == 200:
+                response = HttpResponse(page_response.content, mimetype="text/html; charset=utf-8")
+                response['Content-Encoding'] = 'gzip'
+                response['Last-Modified'] = page_response.headers.get('Last-modified')
+                response['Etag'] = page_response.headers.get('Etag')
+                response['Content-Length'] = str(len(page_response.content))
+                logging.user(request, "~FYLoading original page, proxied from node: ~SB%s bytes" %
+                             (len(page_response.content)))
+                return response
+        
+        if settings.BACKED_BY_AWS['pages_on_s3'] and feed.s3_page:
             if settings.PROXY_S3_PAGES:
                 key = settings.S3_PAGES_BUCKET.get_key(feed.s3_pages_key)
                 if key:

fabfile.py

@@ -66,7 +66,6 @@ env.roledefs ={
                 'ec2-184-73-2-61.compute-1.amazonaws.com',
                 ],
     'vps': ['task01.newsblur.com', 
-            'task02.newsblur.com', 
             'task03.newsblur.com', 
             'task04.newsblur.com', 
             'task08.newsblur.com', 
@@ -75,7 +74,6 @@ env.roledefs ={
             'task11.newsblur.com', 
             'app01.newsblur.com', 
             'app02.newsblur.com', 
-            'app03.newsblur.com',
             ],
 }
 
@@ -253,6 +251,7 @@ def backup_mongo():
         run('python backup_mongo.py')
 
 def backup_postgresql():
+    # crontab for postgres master server
     # 0 4 * * * python /home/sclay/newsblur/utils/backups/backup_psql.py
     # 0 * * * * sudo find /var/lib/postgresql/9.1/archive -mtime +1 -exec rm {} \;
     with cd(os.path.join(env.NEWSBLUR_PATH, 'utils/backups')):

node/original_page.coffee

@@ -5,7 +5,9 @@ mkdirp = require 'mkdirp'
 
 app = express.createServer()
 app.use express.bodyParser()
-    
+
+app.listen 3060
+
 app.get /^\/original_page\/(\d+)\/?/, (req, res) =>
     feedId = parseInt(req.params, 10)
     etag = req.header('If-None-Match')
@@ -14,7 +16,8 @@ app.get /^\/original_page\/(\d+)\/?/, (req, res) =>
     filePath = "originals/#{feedIdDir}.zhtml"
     
     path.exists filePath, (exists, err) ->
-        console.log "Req: #{feedId} (#{filePath}), etag: #{etag}"
+        console.log " ---> Loading: #{feedId} (#{filePath}). " +
+                    "#{if exists then "" else "NOT FOUND"}"
         if not exists
             return res.send 404
         fs.stat filePath, (err, stats) ->
@@ -27,6 +30,7 @@ app.get /^\/original_page\/(\d+)\/?/, (req, res) =>
                 res.header 'Etag', Date.parse(stats.mtime)
                 res.send content
 
+
 app.post /^\/original_page\/(\d+)\/?/, (req, res) =>
     feedId = parseInt(req.params, 10)
     feedIdDir = splitFeedId feedId
@@ -38,7 +42,8 @@ app.post /^\/original_page\/(\d+)\/?/, (req, res) =>
             console.log err if err
             console.log " ---> Saving: #{feedId} (#{filePath})"
             res.send "OK"
-    
+
+
 splitFeedId = (feedId) ->
     feedId += ''
     # x2 = if feedId.length > 1 then '.' + feedId[1] else ''
@@ -46,4 +51,3 @@ splitFeedId = (feedId) ->
     feedId = feedId.replace rgx, '$1' + '/' + '$2' while rgx.test(feedId)
     return feedId;
     
-app.listen 3060

node/original_page.js

@@ -15,6 +15,8 @@
 
   app.use(express.bodyParser());
 
+  app.listen(3060);
+
   app.get(/^\/original_page\/(\d+)\/?/, function(req, res) {
     var etag, feedId, feedIdDir, filePath, lastModified;
     feedId = parseInt(req.params, 10);
@@ -23,7 +25,7 @@
     feedIdDir = splitFeedId(feedId);
     filePath = "originals/" + feedIdDir + ".zhtml";
     return path.exists(filePath, function(exists, err) {
-      console.log("Req: " + feedId + " (" + filePath + "), etag: " + etag);
+      console.log((" ---> Loading: " + feedId + " (" + filePath + "). ") + ("" + (exists ? "" : "NOT FOUND")));
       if (!exists) {
         return res.send(404);
       }
@@ -70,6 +72,4 @@
     return feedId;
   };
 
-  app.listen(3060);
-
 }).call(this);