public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-04-13 09:38:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rewriting original page server to use async.

Browse source
This commit is contained in:
Samuel Clay 2024-02-25 17:13:38 -05:00
parent 9c15c1ac60
commit d6643bb8ab
6 changed files with 91 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -58,6 +58,7 @@ docker/postgres/postgres.conf
# Local configuration file (sdk path, etc)
/originals
/node/originals
media/safari/NewsBlur.safariextz
# IDE files

4
ansible/roles/redis-exporter/tasks/main.yml
View file

@ -18,7 +18,7 @@
redis_target: "redis-user"
redis_port: 6381
- port: 9122
redis_target: "redis-sessions"
redis_target: "redis-session"
redis_port: 6382
- port: 9123
redis_target: "redis-story"
@ -39,7 +39,7 @@
- port: 9121
redis_target: "redis-user"
- port: 9122
redis_target: "redis-sessions"
redis_target: "redis-session"
- port: 9123
redis_target: "redis-story"
- port: 9124

6
apps/reader/views.py
View file

@ -880,7 +880,6 @@ def load_feed_page(request, feed_id):
raise Http404
feed = Feed.get_by_id(feed_id)
if feed and feed.has_page and not feed.has_page_exception:
if settings.BACKED_BY_AWS.get('pages_on_node'):
domain = Site.objects.get_current().domain
@ -889,8 +888,9 @@ def load_feed_page(request, feed_id):
feed.pk,
)
try:
page_response = requests.get(url)
except requests.ConnectionError:
page_response = requests.get(url, verify=not settings.DEBUG)
except requests.ConnectionError as e:
logging.user(request, f"~FR~SBError loading original page: {url} {e}")
page_response = None
if page_response and page_response.status_code == 200:
response = HttpResponse(page_response.content, content_type="text/html; charset=utf-8")

1
docker-compose.yml
View file

@ -50,6 +50,7 @@ services:
- 8008:8008
volumes:
- ${PWD}/node:/srv
- ${PWD}/node/originals:/srv/originals
imageproxy:
container_name: imageproxy

76
node/original_page.coffee
View file

@ -4,56 +4,66 @@ fs = require 'fs'
mkdirp = require 'mkdirp'
log = require './log.js'
fsPromises = fs.promises
mkdirpPromise = require('util').promisify(mkdirp)
original_page = (app) =>
DEV = process.env.NODE_ENV == 'development' || process.env.NODE_ENV == 'docker' || process.env.NODE_ENV == 'debug'
DB_PATH = if DEV then 'originals' else '/srv/originals'
DB_PATH = '/srv/originals'
app.use busboy()
app.get /^\/original_page\/(\d+)\/?/, (req, res) =>
if req.query.test
return res.end "OK"
app.get /^\/original_page\/(\d+)\/?/, (req, res) ->
return res.end "OK" if req.query.test
feedId = parseInt(req.params[0], 10)
etag = req.header('If-None-Match')
lastModified = req.header('If-Modified-Since')
feedId = parseInt req.params[0], 10
etag = req.header 'If-None-Match'
lastModified = req.header 'If-Modified-Since'
feedIdDir = splitFeedId feedId
filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
fs.exists filePath, (exists, err) ->
log.debug "Loading: #{feedId} (#{filePath}). " +
"#{if exists then "" else "NOT FOUND"}"
if not exists
return res.sendStatus 404
fs.stat filePath, (err, stats) ->
if not err and etag and stats.mtime == etag
return res.sendStatus 304
if not err and lastModified and stats.mtime == lastModified
return res.sendStatus 304
fs.readFile filePath, (err, content) ->
res.header 'Etag', Date.parse(stats.mtime)
# Convert to async flow with try/catch using CoffeeScript's then/catch for Promises
fsPromises.stat(filePath).then (stats) ->
fileEtag = Date.parse(stats.mtime).toString()
if etag is fileEtag or lastModified is stats.mtime.toISOString()
log.debug "Not modified: #{feedId} (#{filePath})"
res.sendStatus 304
else
fsPromises.readFile(filePath).then (content) ->
log.debug "Sending: #{feedId} (#{filePath}) #{stats.size} bytes"
res.header 'Etag', fileEtag
res.send content
.catch (err) ->
if err.code is 'ENOENT'
log.debug "Original page not found: #{feedId} (#{filePath})"
res.sendStatus 404
else
log.debug "Error reading original page: #{feedId} (#{filePath}) #{err}"
res.sendStatus 500
app.post /^\/original_page\/(\d+)\/?/, (req, res) =>
feedId = parseInt(req.params[0], 10)
app.post /^\/original_page\/(\d+)\/?/, (req, res) ->
feedId = parseInt req.params[0], 10
feedIdDir = splitFeedId feedId
req.pipe req.busboy
req.busboy.on 'file', (fieldname, file, filename) ->
# log.debug "Uploading #{fieldname} / #{file} / #{filename}"
filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
filePathDir = path.dirname filePath
mkdirp filePathDir, (err) ->
log.debug err if err
fstream = fs.createWriteStream filePath
filePath = "#{DB_PATH}/#{feedIdDir}.zhtml"
filePathDir = path.dirname filePath
# Ensure directory exists before proceeding
mkdirpPromise(filePathDir).then ->
fstream = fs.createWriteStream filePath
req.pipe req.busboy
req.busboy.on 'file', (fieldname, file, filename) ->
file.pipe fstream
fstream.on 'close', ->
fs.stat filePath, (err, stats) ->
log.debug err if err
fsPromises.stat(filePath).then (stats) ->
log.debug "Saving: #{feedId} (#{filePath}) #{stats.size} bytes"
res.send "OK"
.catch (err) ->
log.debug err
res.sendStatus 500
splitFeedId = (feedId) ->

77
node/original_page.js
View file

@ -1,6 +1,6 @@
// Generated by CoffeeScript 2.6.1
(function() {
var busboy, fs, log, mkdirp, original_page, path;
var busboy, fs, fsPromises, log, mkdirp, mkdirpPromise, original_page, path;
path = require('path');
@ -12,12 +12,16 @@
log = require('./log.js');
fsPromises = fs.promises;
mkdirpPromise = require('util').promisify(mkdirp);
original_page = (app) => {
var DB_PATH, DEV, splitFeedId;
DEV = process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'docker' || process.env.NODE_ENV === 'debug';
DB_PATH = DEV ? 'originals' : '/srv/originals';
DB_PATH = '/srv/originals';
app.use(busboy());
app.get(/^\/original_page\/(\d+)\/?/, (req, res) => {
app.get(/^\/original_page\/(\d+)\/?/, function(req, res) {
var etag, feedId, feedIdDir, filePath, lastModified;
if (req.query.test) {
return res.end("OK");
@ -27,52 +31,53 @@
lastModified = req.header('If-Modified-Since');
feedIdDir = splitFeedId(feedId);
filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
return fs.exists(filePath, function(exists, err) {
log.debug(`Loading: ${feedId} (${filePath}). ` + `${exists ? "" : "NOT FOUND"}`);
if (!exists) {
return res.sendStatus(404);
}
return fs.stat(filePath, function(err, stats) {
if (!err && etag && stats.mtime === etag) {
return res.sendStatus(304);
}
if (!err && lastModified && stats.mtime === lastModified) {
return res.sendStatus(304);
}
return fs.readFile(filePath, function(err, content) {
res.header('Etag', Date.parse(stats.mtime));
// Convert to async flow with try/catch using CoffeeScript's then/catch for Promises
return fsPromises.stat(filePath).then(function(stats) {
var fileEtag;
fileEtag = Date.parse(stats.mtime).toString();
if (etag === fileEtag || lastModified === stats.mtime.toISOString()) {
log.debug(`Not modified: ${feedId} (${filePath})`);
return res.sendStatus(304);
} else {
return fsPromises.readFile(filePath).then(function(content) {
log.debug(`Sending: ${feedId} (${filePath}) ${stats.size} bytes`);
res.header('Etag', fileEtag);
return res.send(content);
});
});
}
}).catch(function(err) {
if (err.code === 'ENOENT') {
log.debug(`Original page not found: ${feedId} (${filePath})`);
return res.sendStatus(404);
} else {
log.debug(`Error reading original page: ${feedId} (${filePath}) ${err}`);
return res.sendStatus(500);
}
});
});
app.post(/^\/original_page\/(\d+)\/?/, (req, res) => {
var feedId, feedIdDir;
app.post(/^\/original_page\/(\d+)\/?/, function(req, res) {
var feedId, feedIdDir, filePath, filePathDir;
feedId = parseInt(req.params[0], 10);
feedIdDir = splitFeedId(feedId);
req.pipe(req.busboy);
return req.busboy.on('file', function(fieldname, file, filename) {
var filePath, filePathDir;
// log.debug "Uploading #{fieldname} / #{file} / #{filename}"
filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
filePathDir = path.dirname(filePath);
return mkdirp(filePathDir, function(err) {
var fstream;
if (err) {
log.debug(err);
}
fstream = fs.createWriteStream(filePath);
filePath = `${DB_PATH}/${feedIdDir}.zhtml`;
filePathDir = path.dirname(filePath);
// Ensure directory exists before proceeding
return mkdirpPromise(filePathDir).then(function() {
var fstream;
fstream = fs.createWriteStream(filePath);
req.pipe(req.busboy);
return req.busboy.on('file', function(fieldname, file, filename) {
file.pipe(fstream);
return fstream.on('close', function() {
return fs.stat(filePath, function(err, stats) {
if (err) {
log.debug(err);
}
return fsPromises.stat(filePath).then(function(stats) {
log.debug(`Saving: ${feedId} (${filePath}) ${stats.size} bytes`);
return res.send("OK");
});
});
});
}).catch(function(err) {
log.debug(err);
return res.sendStatus(500);
});
});
splitFeedId = function(feedId) {