diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 64eafd8a8..37f9f29a8 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1073,6 +1073,7 @@ class Feed(models.Model): now = datetime.datetime.now() month_ago = now - datetime.timedelta(days=settings.DAYS_OF_UNREAD_NEW) feed_count = Feed.objects.latest('pk').pk + total = 0 for feed_id in xrange(start, feed_count): if feed_id % 1000 == 0: print "\n\n -------------------------- %s --------------------------\n\n" % feed_id @@ -1090,7 +1091,9 @@ class Feed(models.Model): if dryrun: print " DRYRUN: %s cutoff - %s" % (cutoff, feed) else: - MStory.trim_feed(feed=feed, cutoff=cutoff, verbose=verbose) + total += MStory.trim_feed(feed=feed, cutoff=cutoff, verbose=verbose) + + print " ---> Deleted %s stories in total." % total @property def story_cutoff(self): @@ -1656,8 +1659,9 @@ class MStory(mongo.Document): @classmethod def trim_feed(cls, cutoff, feed_id=None, feed=None, verbose=True): + extra_stories_count = 0 if not feed_id and not feed: - return + return extra_stories_count if not feed_id: feed_id = feed.pk @@ -1675,7 +1679,7 @@ class MStory(mongo.Document): story_trim_date = stories[cutoff].story_date except IndexError, e: logging.debug(' ***> [%-30s] ~BRError trimming feed: %s' % (unicode(feed)[:30], e)) - return + return extra_stories_count extra_stories = MStory.objects(story_feed_id=feed_id, story_date__lte=story_trim_date) @@ -1687,6 +1691,8 @@ class MStory(mongo.Document): logging.debug(" ---> Deleted %s stories, %s left." % ( extra_stories_count, existing_story_count)) + + return extra_stories_count @classmethod def find_story(cls, story_feed_id, story_id, original_only=False): diff --git a/fabfile.py b/fabfile.py index 15f1a2137..692a13a0c 100644 --- a/fabfile.py +++ b/fabfile.py @@ -9,6 +9,7 @@ from fabric.contrib import django from fabric.state import connections from vendor import yaml from pprint import pprint +from collections import defaultdict import os import time import sys @@ -35,6 +36,7 @@ env.NEWSBLUR_PATH = "~/projects/newsblur" env.SECRETS_PATH = "~/projects/secrets-newsblur" env.VENDOR_PATH = "~/projects/code" env.user = 'sclay' +env.key_filename = os.path.join(env.SECRETS_PATH, 'keys/newsblur.key') # ========= # = Roles = @@ -77,8 +79,24 @@ def do_roledefs(split=False): def list_do(): droplets = do(split=True) pprint(droplets) + doapi = dop.client.Client(django_settings.DO_CLIENT_KEY, django_settings.DO_API_KEY) + droplets = doapi.show_active_droplets() + sizes = doapi.sizes() + sizes = dict((size.id, re.split(r"([^0-9]+)", size.name)[0]) for size in sizes) + role_costs = defaultdict(int) + total_cost = 0 + for droplet in droplets: + roledef = re.split(r"([0-9]+)", droplet.name)[0] + cost = int(sizes[droplet.size_id]) * 10 + role_costs[roledef] += cost + total_cost += cost + + print "\n\n Costs:" + pprint(dict(role_costs)) + print " ---> Total cost: $%s/month" % total_cost + + - def host(*names): env.hosts = [] hostnames = do(split=True) @@ -133,6 +151,10 @@ def node(): do() env.roles = ['node'] +def push(): + do() + env.roles = ['push'] + def db(): do() env.roles = ['db'] @@ -327,12 +349,14 @@ def setup_user(): run('ssh-keygen -t dsa -f ~/.ssh/id_dsa -N ""') run('touch ~/.ssh/authorized_keys') put("~/.ssh/id_dsa.pub", "authorized_keys") - run('echo `cat authorized_keys` >> ~/.ssh/authorized_keys') + run("echo \"\n\" >> ~sclay/.ssh/authorized_keys") + run('echo `cat authorized_keys` >> ~sclay/.ssh/authorized_keys') run('rm authorized_keys') def copy_ssh_keys(): put(os.path.join(env.SECRETS_PATH, 'keys/newsblur.key.pub'), "local_keys") - run("echo `\ncat local_keys` >> .ssh/authorized_keys") + run("echo \"\n\" >> ~sclay/.ssh/authorized_keys") + run("echo `cat local_keys` >> ~sclay/.ssh/authorized_keys") run("rm local_keys") def setup_repo(): @@ -525,6 +549,10 @@ def setup_ulimit(): # echo "net.ipv4.ip_local_port_range = 1024 65535" >> /etc/sysctl.conf # sudo chmod 644 /etc/sysctl.conf +def setup_syncookies(): + sudo('echo 1 > /proc/sys/net/ipv4/tcp_syncookies') + sudo('sudo /sbin/sysctl -w net.ipv4.tcp_syncookies=1') + def setup_sudoers(user=None): sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % (user or env.user)) @@ -852,6 +880,7 @@ def setup_redis(slave=False): sudo('update-rc.d redis defaults') sudo('/etc/init.d/redis stop') sudo('/etc/init.d/redis start') + setup_syncookies() def setup_munin(): # sudo('apt-get update') @@ -982,7 +1011,7 @@ def setup_do(name, size=2, image=None): doapi = dop.client.Client(django_settings.DO_CLIENT_KEY, django_settings.DO_API_KEY) sizes = dict((s.name, s.id) for s in doapi.sizes()) size_id = sizes[INSTANCE_SIZE] - ssh_key_id = doapi.all_ssh_keys()[0].id + ssh_key_ids = [str(k.id) for k in doapi.all_ssh_keys()] region_id = doapi.regions()[0].id if not image: IMAGE_NAME = "Ubuntu 13.04 x64" @@ -997,7 +1026,7 @@ def setup_do(name, size=2, image=None): size_id=size_id, image_id=image_id, region_id=region_id, - ssh_key_ids=[str(ssh_key_id)], + ssh_key_ids=ssh_key_ids, virtio=True) print "Booting droplet: %s/%s (size: %s)" % (instance.id, IMAGE_NAME, INSTANCE_SIZE) diff --git a/local_settings.py.template b/local_settings.py.template index 36b2cc01c..f4d2bc198 100644 --- a/local_settings.py.template +++ b/local_settings.py.template @@ -98,6 +98,12 @@ CELERY_RESULT_BACKEND = BROKER_URL REDIS = { 'host': '127.0.0.1', } +REDIS_PUBSUB = { + 'host': '127.0.0.1', +} +REDIS_STORY = { + 'host': '127.0.0.1', +} ELASTICSEARCH_HOSTS = ['127.0.0.1:9200'] diff --git a/media/css/reader.css b/media/css/reader.css index 6bf6c928c..1665f7909 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -1586,7 +1586,7 @@ background: transparent; max-width: 56px; } .NB-story-pane-west #story_titles .NB-storytitles-shares { - top: none; + top: inherit; bottom: 4px; } #story_titles .NB-storytitles-shares .NB-icon { @@ -6125,6 +6125,10 @@ form.opml_import_form input { .NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-pinboard { background: transparent url('/media/embed/reader/pinboard.png') no-repeat 0 0; } +.NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-buffer { + background: transparent url('/media/embed/reader/buffer.png') no-repeat 0 0; + background-size: 16px; +} .NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-diigo { background: transparent url('/media/embed/reader/diigo.png') no-repeat 0 0; } @@ -6145,7 +6149,8 @@ form.opml_import_form input { background-size: 16px; } .NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-twitter { - background: transparent url('/media/embed/reader/twitter_icon.png') no-repeat 0 0; + background: transparent url('/media/embed/reader/twitter_bird.png') no-repeat 0 0; + background-size: 16px; } .NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-facebook { background: transparent url('/media/embed/reader/facebook_icon.png') no-repeat 0 0; @@ -6193,6 +6198,13 @@ form.opml_import_form input { .NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-pinboard .NB-menu-manage-thirdparty-pinboard { opacity: 1; } +.NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-buffer .NB-menu-manage-image, +.NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-buffer .NB-menu-manage-thirdparty-icon { + opacity: .2; +} +.NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-buffer .NB-menu-manage-thirdparty-buffer { + opacity: 1; +} .NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-diigo .NB-menu-manage-image, .NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-diigo .NB-menu-manage-thirdparty-icon { opacity: .2; @@ -8359,7 +8371,8 @@ form.opml_import_form input { margin: 0 0 0 2px; } .NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-twitter] { - background: transparent url('/media/embed/reader/twitter.png') no-repeat 0 0; + background: transparent url('/media/embed/reader/twitter_bird.png') no-repeat 0 0; + background-size: 16px; } .NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-facebook] { background: transparent url('/media/embed/reader/facebook.gif') no-repeat 0 0; @@ -8376,6 +8389,10 @@ form.opml_import_form input { .NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-pinboard] { background: transparent url('/media/embed/reader/pinboard.png') no-repeat 0 0; } +.NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-buffer] { + background: transparent url('/media/embed/reader/buffer.png') no-repeat 0 0; + background-size: 16px; +} .NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-diigo] { background: transparent url('/media/embed/reader/diigo.png') no-repeat 0 0; } diff --git a/media/img/reader/buffer.png b/media/img/reader/buffer.png new file mode 100644 index 000000000..860e30a05 Binary files /dev/null and b/media/img/reader/buffer.png differ diff --git a/media/js/newsblur/reader/reader.js b/media/js/newsblur/reader/reader.js index 523d2ffea..03428d908 100644 --- a/media/js/newsblur/reader/reader.js +++ b/media/js/newsblur/reader/reader.js @@ -2096,6 +2096,20 @@ NEWSBLUR.assets.stories.mark_read(story, {skip_delay: true}); }, + send_story_to_buffer: function(story_id) { + var story = this.model.get_story(story_id); + var url = 'https://bufferapp.com/add?source=newsblur&'; + var buffer_url = [ + url, + 'url=', + encodeURIComponent(story.get('story_permalink')), + '&text=', + encodeURIComponent(story.get('story_title')) + ].join(''); + window.open(buffer_url, '_blank'); + NEWSBLUR.assets.stories.mark_read(story, {skip_delay: true}); + }, + send_story_to_diigo: function(story_id) { var story = this.model.get_story(story_id); var url = 'http://www.diigo.com/post?'; @@ -3008,6 +3022,11 @@ }, this)).bind('mouseleave', _.bind(function(e) { $(e.target).siblings('.NB-menu-manage-title').text('Email story').parent().removeClass('NB-menu-manage-highlight-pinboard'); }, this))), + (NEWSBLUR.Preferences['story_share_buffer'] && $.make('div', { className: 'NB-menu-manage-thirdparty-icon NB-menu-manage-thirdparty-buffer'}).bind('mouseenter', _.bind(function(e) { + $(e.target).siblings('.NB-menu-manage-title').text('Buffer').parent().addClass('NB-menu-manage-highlight-buffer'); + }, this)).bind('mouseleave', _.bind(function(e) { + $(e.target).siblings('.NB-menu-manage-title').text('Email story').parent().removeClass('NB-menu-manage-highlight-buffer'); + }, this))), (NEWSBLUR.Preferences['story_share_diigo'] && $.make('div', { className: 'NB-menu-manage-thirdparty-icon NB-menu-manage-thirdparty-diigo'}).bind('mouseenter', _.bind(function(e) { $(e.target).siblings('.NB-menu-manage-title').text('Diigo').parent().addClass('NB-menu-manage-highlight-diigo'); }, this)).bind('mouseleave', _.bind(function(e) { @@ -3058,6 +3077,8 @@ this.send_story_to_readability(story.id); } else if ($target.hasClass('NB-menu-manage-thirdparty-pinboard')) { this.send_story_to_pinboard(story.id); + } else if ($target.hasClass('NB-menu-manage-thirdparty-buffer')) { + this.send_story_to_buffer(story.id); } else if ($target.hasClass('NB-menu-manage-thirdparty-diigo')) { this.send_story_to_diigo(story.id); } else if ($target.hasClass('NB-menu-manage-thirdparty-kippt')) { diff --git a/media/js/newsblur/reader/reader_preferences.js b/media/js/newsblur/reader/reader_preferences.js index d2cc05d24..7e7a191cb 100644 --- a/media/js/newsblur/reader/reader_preferences.js +++ b/media/js/newsblur/reader/reader_preferences.js @@ -473,6 +473,10 @@ _.extend(NEWSBLUR.ReaderPreferences.prototype, { $.make('input', { type: 'checkbox', id: 'NB-preference-story-share-pinboard', name: 'story_share_pinboard' }), $.make('label', { 'for': 'NB-preference-story-share-pinboard' }) ]), + $.make('div', { className: 'NB-preference-option', title: 'Buffer' }, [ + $.make('input', { type: 'checkbox', id: 'NB-preference-story-share-buffer', name: 'story_share_buffer' }), + $.make('label', { 'for': 'NB-preference-story-share-buffer' }) + ]), $.make('div', { className: 'NB-preference-option', title: 'Diigo' }, [ $.make('input', { type: 'checkbox', id: 'NB-preference-story-share-diigo', name: 'story_share_diigo' }), $.make('label', { 'for': 'NB-preference-story-share-diigo' }) diff --git a/node/unread_counts.coffee b/node/unread_counts.coffee index f15324540..bfaa11902 100644 --- a/node/unread_counts.coffee +++ b/node/unread_counts.coffee @@ -2,7 +2,7 @@ fs = require 'fs' redis = require 'redis' log = require './log.js' -REDIS_SERVER = if process.env.NODE_ENV == 'development' then 'localhost' else 'db13' +REDIS_SERVER = if process.env.NODE_ENV == 'development' then 'localhost' else 'db_redis_pubsub' SECURE = !!process.env.NODE_SSL # client = redis.createClient 6379, REDIS_SERVER @@ -49,10 +49,16 @@ io.sockets.on 'connection', (socket) -> if not @username return + socket.on "error", (err) -> + console.log " ---> Error (socket): #{err}" socket.subscribe?.end() socket.subscribe = redis.createClient 6379, REDIS_SERVER - socket.subscribe.subscribe @feeds - socket.subscribe.subscribe @username + socket.subscribe.on "error", (err) -> + console.log " ---> Error: #{err}" + socket.subscribe.end() + socket.subscribe.on "connect", => + socket.subscribe.subscribe @feeds + socket.subscribe.subscribe @username socket.subscribe.on 'message', (channel, message) => log.info @username, "Update on #{channel}: #{message}" @@ -66,3 +72,6 @@ io.sockets.on 'connection', (socket) -> log.info @username, "Disconnect (#{@feeds?.length} feeds, #{ip})," + " there are now #{io.sockets.clients().length-1} users. " + " #{if SECURE then "(SSL)" else "(non-SSL)"}" + +io.sockets.on 'error', (err) -> + console.log " ---> Error (sockets): #{err}" diff --git a/node/unread_counts.js b/node/unread_counts.js index de2c307a2..4fde4e39a 100644 --- a/node/unread_counts.js +++ b/node/unread_counts.js @@ -8,7 +8,7 @@ log = require('./log.js'); - REDIS_SERVER = process.env.NODE_ENV === 'development' ? 'localhost' : 'db13'; + REDIS_SERVER = process.env.NODE_ENV === 'development' ? 'localhost' : 'db_redis_pubsub'; SECURE = !!process.env.NODE_SSL; @@ -48,12 +48,21 @@ if (!this.username) { return; } + socket.on("error", function(err) { + return console.log(" ---> Error (socket): " + err); + }); if ((_ref = socket.subscribe) != null) { _ref.end(); } socket.subscribe = redis.createClient(6379, REDIS_SERVER); - socket.subscribe.subscribe(this.feeds); - socket.subscribe.subscribe(this.username); + socket.subscribe.on("error", function(err) { + console.log(" ---> Error: " + err); + return socket.subscribe.end(); + }); + socket.subscribe.on("connect", function() { + socket.subscribe.subscribe(_this.feeds); + return socket.subscribe.subscribe(_this.username); + }); return socket.subscribe.on('message', function(channel, message) { log.info(_this.username, "Update on " + channel + ": " + message); if (channel === _this.username) { @@ -72,4 +81,8 @@ }); }); + io.sockets.on('error', function(err) { + return console.log(" ---> Error (sockets): " + err); + }); + }).call(this); diff --git a/settings.py b/settings.py index 4fc0669f1..569f6fbda 100644 --- a/settings.py +++ b/settings.py @@ -66,6 +66,7 @@ SITE_ID = 1 USE_I18N = False LOGIN_REDIRECT_URL = '/' LOGIN_URL = '/reader/login' +MEDIA_URL = '/media/' # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a # trailing slash. # Examples: "http://foo.com/media/", "/media/". @@ -199,7 +200,7 @@ INTERNAL_IPS = ('127.0.0.1',) LOGGING_LOG_SQL = True APPEND_SLASH = False SOUTH_TESTS_MIGRATE = False -SESSION_ENGINE = "django.contrib.sessions.backends.db" +SESSION_ENGINE = 'redis_sessions.session' TEST_RUNNER = "utils.testrunner.TestRunner" SESSION_COOKIE_NAME = 'newsblur_sessionid' SESSION_COOKIE_AGE = 60*60*24*365*2 # 2 years @@ -380,7 +381,7 @@ CELERYBEAT_SCHEDULE = { }, 'activate-next-new-user': { 'task': 'activate-next-new-user', - 'schedule': datetime.timedelta(minutes=5), + 'schedule': datetime.timedelta(minutes=3.5), 'options': {'queue': 'beat_tasks'}, }, } @@ -390,11 +391,11 @@ CELERYBEAT_SCHEDULE = { # ========= MONGO_DB = { - 'host': '127.0.0.1:27017', + 'host': 'db_mongo:27017', 'name': 'newsblur', } MONGO_ANALYTICS_DB = { - 'host': '127.0.0.1:27017', + 'host': 'db_mongo_analytics:27017', 'name': 'nbanalytics', } @@ -429,14 +430,15 @@ class MasterSlaveRouter(object): # ========= REDIS = { - 'host': 'db12', + 'host': 'db_redis', } -REDIS2 = { - 'host': 'db13', +REDIS_PUBSUB = { + 'host': 'db_redis_pubsub', } -REDIS3 = { - 'host': 'db11', +REDIS_STORY = { + 'host': 'db_redis_story', } + CELERY_REDIS_DB = 4 SESSION_REDIS_DB = 5 @@ -444,7 +446,7 @@ SESSION_REDIS_DB = 5 # = Elasticsearch = # ================= -ELASTICSEARCH_HOSTS = ['db01:9200'] +ELASTICSEARCH_HOSTS = ['db_search:9200'] # =============== # = Social APIs = @@ -460,7 +462,7 @@ TWITTER_CONSUMER_SECRET = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' # = AWS Backing = # =============== -ORIGINAL_PAGE_SERVER = "db01:3060" +ORIGINAL_PAGE_SERVER = "db_pages:3060" BACKED_BY_AWS = { 'pages_on_s3': False, @@ -533,6 +535,18 @@ else: BROKER_BACKEND = "redis" BROKER_URL = "redis://%s:6379/%s" % (REDIS['host'], CELERY_REDIS_DB) CELERY_RESULT_BACKEND = BROKER_URL +SESSION_REDIS_HOST = REDIS['host'] + +CACHES = { + 'default': { + 'BACKEND': 'redis_cache.RedisCache', + 'LOCATION': '%s:6379' % REDIS['host'], + 'OPTIONS': { + 'DB': 6, + 'PARSER_CLASS': 'redis.connection.HiredisParser' + }, + }, +} # ========= # = Mongo = @@ -540,7 +554,7 @@ CELERY_RESULT_BACKEND = BROKER_URL MONGO_DB_DEFAULTS = { 'name': 'newsblur', - 'host': 'db02:27017', + 'host': 'db_mongo:27017', 'alias': 'default', } MONGO_DB = dict(MONGO_DB_DEFAULTS, **MONGO_DB) @@ -555,7 +569,7 @@ MONGODB = connect(MONGO_DB.pop('name'), **MONGO_DB) MONGO_ANALYTICS_DB_DEFAULTS = { 'name': 'nbanalytics', - 'host': 'db30:27017', + 'host': 'db_mongo_analytics:27017', 'alias': 'nbanalytics', } MONGO_ANALYTICS_DB = dict(MONGO_ANALYTICS_DB_DEFAULTS, **MONGO_ANALYTICS_DB) @@ -572,11 +586,11 @@ REDIS_STATISTICS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=3 REDIS_FEED_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=4) REDIS_SESSION_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=5) # REDIS_CACHE_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=6) # Duped in CACHES -REDIS_STORY_HASH_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=8) +REDIS_STORY_HASH_POOL = redis.ConnectionPool(host=REDIS_STORY['host'], port=6379, db=1) -REDIS_PUBSUB_POOL = redis.ConnectionPool(host=REDIS2['host'], port=6379, db=0) +REDIS_PUBSUB_POOL = redis.ConnectionPool(host=REDIS_PUBSUB['host'], port=6379, db=0) -REDIS_STORY_HASH_POOL2 = redis.ConnectionPool(host=REDIS3['host'], port=6379, db=1) +REDIS_STORY_HASH_POOL2 = redis.ConnectionPool(host=REDIS['host'], port=6379, db=8) # ========== # = Assets = diff --git a/utils/backups/backup_redis_story.py b/utils/backups/backup_redis_story.py new file mode 100644 index 000000000..c4909dbd8 --- /dev/null +++ b/utils/backups/backup_redis_story.py @@ -0,0 +1,16 @@ +import os +import sys + +CURRENT_DIR = os.path.dirname(__file__) +NEWSBLUR_DIR = ''.join([CURRENT_DIR, '/../../']) +sys.path.insert(0, NEWSBLUR_DIR) +os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' + +import time +import s3 +from django.conf import settings + +filename = 'backup_redis_story_%s.rdb.gz' % time.strftime('%Y-%m-%d-%H-%M') +path = '/var/lib/redis/dump.rdb' +print 'Uploading %s (from %s) to S3...' % (filename, path) +s3.save_file_in_s3(path, name=filename)