From ed81559e4e906e79b34caa79b2f4530f3f7e7ceb Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 25 Dec 2019 18:13:29 -0500 Subject: [PATCH] Adding imageproxy and switching from camo. Built for #1162. --- apps/rss_feeds/models.py | 17 +++++++++++++---- config/camo.nginx.conf | 2 +- config/supervisor_imageproxy.conf | 11 +++++++++++ config/zshrc | 3 ++- fabfile.py | 20 ++++++++++++++++++++ settings.py | 2 +- utils/story_functions.py | 24 +++++++++++++++++++++++- 7 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 config/supervisor_imageproxy.conf diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 02e20b276..9e55756b3 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -47,7 +47,7 @@ from utils.feed_functions import relative_timesince from utils.feed_functions import seconds_timesince from utils.story_functions import strip_tags, htmldiff, strip_comments, strip_comments__lxml from utils.story_functions import prep_for_search -from utils.story_functions import create_camo_signed_url +from utils.story_functions import create_imageproxy_signed_url ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4) @@ -1897,6 +1897,7 @@ class Feed(models.Model): story['story_permalink'] = story_db.story_permalink story['image_urls'] = story_db.image_urls story['secure_image_urls']= cls.secure_image_urls(story_db.image_urls) + story['secure_image_thumbnails']= cls.secure_image_thumbnails(story_db.image_urls) story['story_feed_id'] = feed_id or story_db.story_feed_id story['has_modifications']= has_changes story['comment_count'] = story_db.comment_count if hasattr(story_db, 'comment_count') else 0 @@ -1930,9 +1931,17 @@ class Feed(models.Model): @classmethod def secure_image_urls(cls, urls): - signed_urls = [create_camo_signed_url(settings.IMAGES_URL, - settings.IMAGES_SECRET_KEY, - url) for url in urls] + signed_urls = [create_imageproxy_signed_url(settings.IMAGES_URL, + settings.IMAGES_SECRET_KEY, + url) for url in urls] + return dict(zip(urls, signed_urls)) + + @classmethod + def secure_image_thumbnails(cls, urls, size=200): + signed_urls = [create_imageproxy_signed_url(settings.IMAGES_URL, + settings.IMAGES_SECRET_KEY, + url, + size) for url in urls] return dict(zip(urls, signed_urls)) def get_tags(self, entry): diff --git a/config/camo.nginx.conf b/config/camo.nginx.conf index 1eef30e03..a03321417 100644 --- a/config/camo.nginx.conf +++ b/config/camo.nginx.conf @@ -2,7 +2,7 @@ upstream camo_server { server 127.0.0.1:8081 fail_timeout=10 max_fails=3; } upstream imageproxy_server { - server 127.0.0.1:8082 fail_timeout=10 max_fails=3; + server 127.0.0.1:4593 fail_timeout=10 max_fails=3; } server { diff --git a/config/supervisor_imageproxy.conf b/config/supervisor_imageproxy.conf new file mode 100644 index 000000000..2383538a6 --- /dev/null +++ b/config/supervisor_imageproxy.conf @@ -0,0 +1,11 @@ +[program:imageproxy] +command=/srv/code/go/bin/imageproxy +directory=/srv/code/go/bin/ +user=sclay +autostart=true +autorestart=true +#redirect_stderr=True +priority=991 +stopsignal=HUP +stdout_logfile = /srv/newsblur/logs/imageproxy.log +environment = IMAGEPROXY_CACHE="/var/cache/imageproxy",IMAGEPROXY_ADDR="localhost:4593" \ No newline at end of file diff --git a/config/zshrc b/config/zshrc index 406e962c7..00b33d581 100644 --- a/config/zshrc +++ b/config/zshrc @@ -12,7 +12,8 @@ source $ZSH/oh-my-zsh.sh export PYTHONSTARTUP=$HOME/.pystartup export LSCOLORS='gxgxcxdxBxegedabagacad' - +export GOROOT=/srv/code/go +export GOPATH=/srv/code/go zstyle ':completion:*' matcher-list 'm:{a-z}={A-Z}' zstyle ':completion:*' list-colors "=(#b) #([0-9]#)*=36=31" setopt COMPLETE_IN_WORD diff --git a/fabfile.py b/fabfile.py index 24a29bbac..ef84f4bc0 100644 --- a/fabfile.py +++ b/fabfile.py @@ -1363,6 +1363,26 @@ def setup_db_search(): sudo('supervisorctl reread') sudo('supervisorctl update') +def setup_imageproxy(): + # sudo('apt-get update') + # sudo('apt-get install -y golang') + # with cd(env.VENDOR_PATH): + # with settings(warn_only=True): + # run('git clone https://github.com/willnorris/imageproxy.git') + # run('wget https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz') + # run('tar -xzf go1.13.3.linux-amd64.tar.gz') + # run('rm go1.13.3.linux-amd64.tar.gz') + # sudo('rm /usr/bin/go') + # sudo('ln -s /srv/code/go/bin/go /usr/bin/go') + # with cd(os.path.join(env.VENDOR_PATH, 'imageproxy')): + # run('go get willnorris.com/go/imageproxy/cmd/imageproxy') + put('config/supervisor_imageproxy.conf', '/etc/supervisor/conf.d/supervisor_imageproxy.conf', use_sudo=True) + sudo('supervisorctl reread') + sudo('supervisorctl update') + put("config/camo.nginx.conf", "/usr/local/nginx/conf/sites-enabled/camo.conf", use_sudo=True) + sudo("/etc/init.d/nginx restart") + + @parallel def setup_usage_monitor(): sudo('ln -fs %s/utils/monitor_disk_usage.py /etc/cron.daily/monitor_disk_usage' % env.NEWSBLUR_PATH) diff --git a/settings.py b/settings.py index c9e01dcbf..bade71d01 100644 --- a/settings.py +++ b/settings.py @@ -48,7 +48,7 @@ SERVER_NAME = 'newsblur' SERVER_EMAIL = 'server@newsblur.com' HELLO_EMAIL = 'hello@newsblur.com' NEWSBLUR_URL = 'http://www.newsblur.com' -IMAGES_URL = 'https://images.newsblur.com' +IMAGES_URL = 'https://imageproxy.newsblur.com' SECRET_KEY = 'YOUR_SECRET_KEY' IMAGES_SECRET_KEY = "YOUR_SECRET_IMAGE_KEY" diff --git a/utils/story_functions.py b/utils/story_functions.py index ade7bbfb5..50c052a04 100644 --- a/utils/story_functions.py +++ b/utils/story_functions.py @@ -2,6 +2,8 @@ import re import datetime import struct import dateutil +import hashlib +import base64 from random import randint from HTMLParser import HTMLParser from lxml.html.diff import tokenize, fixup_ins_del_tags, htmldiff_tokens @@ -382,4 +384,24 @@ def create_camo_signed_url(base_url, hmac_key, url): hex_url = hexlify(url.encode()).decode() return ('{base}/{signature}/{hex_url}' - .format(base=base_url, signature=signature, hex_url=hex_url)) \ No newline at end of file + .format(base=base_url, signature=signature, hex_url=hex_url)) + +def create_imageproxy_signed_url(base_url, hmac_key, url, options=None): + """Create a imageproxy signed URL for the specified image URL + Args: + base_url: Base URL of the imageproxy installation + hmac_key: HMAC shared key to be used for signing + url: URL of the destination image + Returns: + str: A full url that can be used to serve the proxied image + """ + if not options: options = [] + if isinstance(options, str): options = [options] + base_url = base_url.rstrip('/') + signature = base64.urlsafe_b64encode(hmac.new(hmac_key, msg=url, digestmod=hashlib.sha256).digest()) + options.append(signature) + hex_url = hexlify(url.encode()).decode() + + return ('{base}/{options}/{hex_url}' + .format(base=base_url, options=','.join(options), hex_url=hex_url)) + \ No newline at end of file