public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-09-18 21:50:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adding imageproxy and switching from camo. Built for #1162.

Browse source
This commit is contained in:
Samuel Clay 2019-12-25 18:13:29 -05:00
parent c7091e6772
commit ed81559e4e
7 changed files with 71 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
apps/rss_feeds/models.py
View file

@ -47,7 +47,7 @@ from utils.feed_functions import relative_timesince
from utils.feed_functions import seconds_timesince
from utils.story_functions import strip_tags, htmldiff, strip_comments, strip_comments__lxml
from utils.story_functions import prep_for_search
from utils.story_functions import create_camo_signed_url
from utils.story_functions import create_imageproxy_signed_url
ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
@ -1897,6 +1897,7 @@ class Feed(models.Model):
story['story_permalink'] = story_db.story_permalink
story['image_urls'] = story_db.image_urls
story['secure_image_urls']= cls.secure_image_urls(story_db.image_urls)
story['secure_image_thumbnails']= cls.secure_image_thumbnails(story_db.image_urls)
story['story_feed_id'] = feed_id or story_db.story_feed_id
story['has_modifications']= has_changes
story['comment_count'] = story_db.comment_count if hasattr(story_db, 'comment_count') else 0
@ -1930,9 +1931,17 @@ class Feed(models.Model):
@classmethod
def secure_image_urls(cls, urls):
signed_urls = [create_camo_signed_url(settings.IMAGES_URL,
settings.IMAGES_SECRET_KEY,
url) for url in urls]
signed_urls = [create_imageproxy_signed_url(settings.IMAGES_URL,
settings.IMAGES_SECRET_KEY,
url) for url in urls]
return dict(zip(urls, signed_urls))
@classmethod
def secure_image_thumbnails(cls, urls, size=200):
signed_urls = [create_imageproxy_signed_url(settings.IMAGES_URL,
settings.IMAGES_SECRET_KEY,
url,
size) for url in urls]
return dict(zip(urls, signed_urls))
def get_tags(self, entry):

2
config/camo.nginx.conf
View file

@ -2,7 +2,7 @@ upstream camo_server {
server 127.0.0.1:8081 fail_timeout=10 max_fails=3;
}
upstream imageproxy_server {
server 127.0.0.1:8082 fail_timeout=10 max_fails=3;
server 127.0.0.1:4593 fail_timeout=10 max_fails=3;
}
server {

11
config/supervisor_imageproxy.conf Normal file
View file

@ -0,0 +1,11 @@
[program:imageproxy]
command=/srv/code/go/bin/imageproxy
directory=/srv/code/go/bin/
user=sclay
autostart=true
autorestart=true
#redirect_stderr=True
priority=991
stopsignal=HUP
stdout_logfile = /srv/newsblur/logs/imageproxy.log
environment = IMAGEPROXY_CACHE="/var/cache/imageproxy",IMAGEPROXY_ADDR="localhost:4593"

3
config/zshrc
View file

@ -12,7 +12,8 @@ source $ZSH/oh-my-zsh.sh
export PYTHONSTARTUP=$HOME/.pystartup
export LSCOLORS='gxgxcxdxBxegedabagacad'
export GOROOT=/srv/code/go
export GOPATH=/srv/code/go
zstyle ':completion:*' matcher-list 'm:{a-z}={A-Z}'
zstyle ':completion:*' list-colors "=(#b) #([0-9]#)*=36=31"
setopt COMPLETE_IN_WORD

20
fabfile.py vendored
View file

@ -1363,6 +1363,26 @@ def setup_db_search():
sudo('supervisorctl reread')
sudo('supervisorctl update')
def setup_imageproxy():
# sudo('apt-get update')
# sudo('apt-get install -y golang')
# with cd(env.VENDOR_PATH):
# with settings(warn_only=True):
# run('git clone https://github.com/willnorris/imageproxy.git')
# run('wget https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz')
# run('tar -xzf go1.13.3.linux-amd64.tar.gz')
# run('rm go1.13.3.linux-amd64.tar.gz')
# sudo('rm /usr/bin/go')
# sudo('ln -s /srv/code/go/bin/go /usr/bin/go')
# with cd(os.path.join(env.VENDOR_PATH, 'imageproxy')):
# run('go get willnorris.com/go/imageproxy/cmd/imageproxy')
put('config/supervisor_imageproxy.conf', '/etc/supervisor/conf.d/supervisor_imageproxy.conf', use_sudo=True)
sudo('supervisorctl reread')
sudo('supervisorctl update')
put("config/camo.nginx.conf", "/usr/local/nginx/conf/sites-enabled/camo.conf", use_sudo=True)
sudo("/etc/init.d/nginx restart")
@parallel
def setup_usage_monitor():
sudo('ln -fs %s/utils/monitor_disk_usage.py /etc/cron.daily/monitor_disk_usage' % env.NEWSBLUR_PATH)

2
settings.py
View file

@ -48,7 +48,7 @@ SERVER_NAME = 'newsblur'
SERVER_EMAIL = 'server@newsblur.com'
HELLO_EMAIL = 'hello@newsblur.com'
NEWSBLUR_URL = 'http://www.newsblur.com'
IMAGES_URL = 'https://images.newsblur.com'
IMAGES_URL = 'https://imageproxy.newsblur.com'
SECRET_KEY = 'YOUR_SECRET_KEY'
IMAGES_SECRET_KEY = "YOUR_SECRET_IMAGE_KEY"

24
utils/story_functions.py
View file

@ -2,6 +2,8 @@ import re
import datetime
import struct
import dateutil
import hashlib
import base64
from random import randint
from HTMLParser import HTMLParser
from lxml.html.diff import tokenize, fixup_ins_del_tags, htmldiff_tokens
@ -382,4 +384,24 @@ def create_camo_signed_url(base_url, hmac_key, url):
hex_url = hexlify(url.encode()).decode()
return ('{base}/{signature}/{hex_url}'
.format(base=base_url, signature=signature, hex_url=hex_url))
.format(base=base_url, signature=signature, hex_url=hex_url))
def create_imageproxy_signed_url(base_url, hmac_key, url, options=None):
"""Create a imageproxy signed URL for the specified image URL
Args:
base_url: Base URL of the imageproxy installation
hmac_key: HMAC shared key to be used for signing
url: URL of the destination image
Returns:
str: A full url that can be used to serve the proxied image
"""
if not options: options = []
if isinstance(options, str): options = [options]
base_url = base_url.rstrip('/')
signature = base64.urlsafe_b64encode(hmac.new(hmac_key, msg=url, digestmod=hashlib.sha256).digest())
options.append(signature)
hex_url = hexlify(url.encode()).decode()
return ('{base}/{options}/{hex_url}'
.format(base=base_url, options=','.join(options), hex_url=hex_url))