Merge branch 'master' into tagging

* master:
  Handing out free accounts a little quicker.
  Adding count of trimmed stories.
  Adding redis backup for new redis stroy server.
  Adding Buffer as a share option.
  Moving story hash db to its own server (primary).
  Adding cost to fabfile's list_do.
  Moving node server to db pubsub role.
  Moving from specific db hosts to db roles, editable through /etc/hosts.
  New ssh keys, syn_cookies setting for redis pubsub server.
  Handling socket level errors in socketio/node.
  Catching exceptions for node redis.
  Catching exceptions for node redis.
  Trying to fix issues on node redis connection.
  Handling redis connection timeouts in node.
This commit is contained in:
Samuel Clay 2013-08-13 11:19:23 -07:00
commit e0f9faffb7
11 changed files with 168 additions and 33 deletions

View file

@ -1073,6 +1073,7 @@ class Feed(models.Model):
now = datetime.datetime.now() now = datetime.datetime.now()
month_ago = now - datetime.timedelta(days=settings.DAYS_OF_UNREAD_NEW) month_ago = now - datetime.timedelta(days=settings.DAYS_OF_UNREAD_NEW)
feed_count = Feed.objects.latest('pk').pk feed_count = Feed.objects.latest('pk').pk
total = 0
for feed_id in xrange(start, feed_count): for feed_id in xrange(start, feed_count):
if feed_id % 1000 == 0: if feed_id % 1000 == 0:
print "\n\n -------------------------- %s --------------------------\n\n" % feed_id print "\n\n -------------------------- %s --------------------------\n\n" % feed_id
@ -1090,7 +1091,9 @@ class Feed(models.Model):
if dryrun: if dryrun:
print " DRYRUN: %s cutoff - %s" % (cutoff, feed) print " DRYRUN: %s cutoff - %s" % (cutoff, feed)
else: else:
MStory.trim_feed(feed=feed, cutoff=cutoff, verbose=verbose) total += MStory.trim_feed(feed=feed, cutoff=cutoff, verbose=verbose)
print " ---> Deleted %s stories in total." % total
@property @property
def story_cutoff(self): def story_cutoff(self):
@ -1656,8 +1659,9 @@ class MStory(mongo.Document):
@classmethod @classmethod
def trim_feed(cls, cutoff, feed_id=None, feed=None, verbose=True): def trim_feed(cls, cutoff, feed_id=None, feed=None, verbose=True):
extra_stories_count = 0
if not feed_id and not feed: if not feed_id and not feed:
return return extra_stories_count
if not feed_id: if not feed_id:
feed_id = feed.pk feed_id = feed.pk
@ -1675,7 +1679,7 @@ class MStory(mongo.Document):
story_trim_date = stories[cutoff].story_date story_trim_date = stories[cutoff].story_date
except IndexError, e: except IndexError, e:
logging.debug(' ***> [%-30s] ~BRError trimming feed: %s' % (unicode(feed)[:30], e)) logging.debug(' ***> [%-30s] ~BRError trimming feed: %s' % (unicode(feed)[:30], e))
return return extra_stories_count
extra_stories = MStory.objects(story_feed_id=feed_id, extra_stories = MStory.objects(story_feed_id=feed_id,
story_date__lte=story_trim_date) story_date__lte=story_trim_date)
@ -1687,6 +1691,8 @@ class MStory(mongo.Document):
logging.debug(" ---> Deleted %s stories, %s left." % ( logging.debug(" ---> Deleted %s stories, %s left." % (
extra_stories_count, extra_stories_count,
existing_story_count)) existing_story_count))
return extra_stories_count
@classmethod @classmethod
def find_story(cls, story_feed_id, story_id, original_only=False): def find_story(cls, story_feed_id, story_id, original_only=False):

39
fabfile.py vendored
View file

@ -9,6 +9,7 @@ from fabric.contrib import django
from fabric.state import connections from fabric.state import connections
from vendor import yaml from vendor import yaml
from pprint import pprint from pprint import pprint
from collections import defaultdict
import os import os
import time import time
import sys import sys
@ -35,6 +36,7 @@ env.NEWSBLUR_PATH = "~/projects/newsblur"
env.SECRETS_PATH = "~/projects/secrets-newsblur" env.SECRETS_PATH = "~/projects/secrets-newsblur"
env.VENDOR_PATH = "~/projects/code" env.VENDOR_PATH = "~/projects/code"
env.user = 'sclay' env.user = 'sclay'
env.key_filename = os.path.join(env.SECRETS_PATH, 'keys/newsblur.key')
# ========= # =========
# = Roles = # = Roles =
@ -77,8 +79,24 @@ def do_roledefs(split=False):
def list_do(): def list_do():
droplets = do(split=True) droplets = do(split=True)
pprint(droplets) pprint(droplets)
doapi = dop.client.Client(django_settings.DO_CLIENT_KEY, django_settings.DO_API_KEY)
droplets = doapi.show_active_droplets()
sizes = doapi.sizes()
sizes = dict((size.id, re.split(r"([^0-9]+)", size.name)[0]) for size in sizes)
role_costs = defaultdict(int)
total_cost = 0
for droplet in droplets:
roledef = re.split(r"([0-9]+)", droplet.name)[0]
cost = int(sizes[droplet.size_id]) * 10
role_costs[roledef] += cost
total_cost += cost
print "\n\n Costs:"
pprint(dict(role_costs))
print " ---> Total cost: $%s/month" % total_cost
def host(*names): def host(*names):
env.hosts = [] env.hosts = []
hostnames = do(split=True) hostnames = do(split=True)
@ -133,6 +151,10 @@ def node():
do() do()
env.roles = ['node'] env.roles = ['node']
def push():
do()
env.roles = ['push']
def db(): def db():
do() do()
env.roles = ['db'] env.roles = ['db']
@ -327,12 +349,14 @@ def setup_user():
run('ssh-keygen -t dsa -f ~/.ssh/id_dsa -N ""') run('ssh-keygen -t dsa -f ~/.ssh/id_dsa -N ""')
run('touch ~/.ssh/authorized_keys') run('touch ~/.ssh/authorized_keys')
put("~/.ssh/id_dsa.pub", "authorized_keys") put("~/.ssh/id_dsa.pub", "authorized_keys")
run('echo `cat authorized_keys` >> ~/.ssh/authorized_keys') run("echo \"\n\" >> ~sclay/.ssh/authorized_keys")
run('echo `cat authorized_keys` >> ~sclay/.ssh/authorized_keys')
run('rm authorized_keys') run('rm authorized_keys')
def copy_ssh_keys(): def copy_ssh_keys():
put(os.path.join(env.SECRETS_PATH, 'keys/newsblur.key.pub'), "local_keys") put(os.path.join(env.SECRETS_PATH, 'keys/newsblur.key.pub'), "local_keys")
run("echo `\ncat local_keys` >> .ssh/authorized_keys") run("echo \"\n\" >> ~sclay/.ssh/authorized_keys")
run("echo `cat local_keys` >> ~sclay/.ssh/authorized_keys")
run("rm local_keys") run("rm local_keys")
def setup_repo(): def setup_repo():
@ -525,6 +549,10 @@ def setup_ulimit():
# echo "net.ipv4.ip_local_port_range = 1024 65535" >> /etc/sysctl.conf # echo "net.ipv4.ip_local_port_range = 1024 65535" >> /etc/sysctl.conf
# sudo chmod 644 /etc/sysctl.conf # sudo chmod 644 /etc/sysctl.conf
def setup_syncookies():
sudo('echo 1 > /proc/sys/net/ipv4/tcp_syncookies')
sudo('sudo /sbin/sysctl -w net.ipv4.tcp_syncookies=1')
def setup_sudoers(user=None): def setup_sudoers(user=None):
sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % (user or env.user)) sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % (user or env.user))
@ -852,6 +880,7 @@ def setup_redis(slave=False):
sudo('update-rc.d redis defaults') sudo('update-rc.d redis defaults')
sudo('/etc/init.d/redis stop') sudo('/etc/init.d/redis stop')
sudo('/etc/init.d/redis start') sudo('/etc/init.d/redis start')
setup_syncookies()
def setup_munin(): def setup_munin():
# sudo('apt-get update') # sudo('apt-get update')
@ -982,7 +1011,7 @@ def setup_do(name, size=2, image=None):
doapi = dop.client.Client(django_settings.DO_CLIENT_KEY, django_settings.DO_API_KEY) doapi = dop.client.Client(django_settings.DO_CLIENT_KEY, django_settings.DO_API_KEY)
sizes = dict((s.name, s.id) for s in doapi.sizes()) sizes = dict((s.name, s.id) for s in doapi.sizes())
size_id = sizes[INSTANCE_SIZE] size_id = sizes[INSTANCE_SIZE]
ssh_key_id = doapi.all_ssh_keys()[0].id ssh_key_ids = [str(k.id) for k in doapi.all_ssh_keys()]
region_id = doapi.regions()[0].id region_id = doapi.regions()[0].id
if not image: if not image:
IMAGE_NAME = "Ubuntu 13.04 x64" IMAGE_NAME = "Ubuntu 13.04 x64"
@ -997,7 +1026,7 @@ def setup_do(name, size=2, image=None):
size_id=size_id, size_id=size_id,
image_id=image_id, image_id=image_id,
region_id=region_id, region_id=region_id,
ssh_key_ids=[str(ssh_key_id)], ssh_key_ids=ssh_key_ids,
virtio=True) virtio=True)
print "Booting droplet: %s/%s (size: %s)" % (instance.id, IMAGE_NAME, INSTANCE_SIZE) print "Booting droplet: %s/%s (size: %s)" % (instance.id, IMAGE_NAME, INSTANCE_SIZE)

View file

@ -98,6 +98,12 @@ CELERY_RESULT_BACKEND = BROKER_URL
REDIS = { REDIS = {
'host': '127.0.0.1', 'host': '127.0.0.1',
} }
REDIS_PUBSUB = {
'host': '127.0.0.1',
}
REDIS_STORY = {
'host': '127.0.0.1',
}
ELASTICSEARCH_HOSTS = ['127.0.0.1:9200'] ELASTICSEARCH_HOSTS = ['127.0.0.1:9200']

View file

@ -1586,7 +1586,7 @@ background: transparent;
max-width: 56px; max-width: 56px;
} }
.NB-story-pane-west #story_titles .NB-storytitles-shares { .NB-story-pane-west #story_titles .NB-storytitles-shares {
top: none; top: inherit;
bottom: 4px; bottom: 4px;
} }
#story_titles .NB-storytitles-shares .NB-icon { #story_titles .NB-storytitles-shares .NB-icon {
@ -6125,6 +6125,10 @@ form.opml_import_form input {
.NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-pinboard { .NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-pinboard {
background: transparent url('/media/embed/reader/pinboard.png') no-repeat 0 0; background: transparent url('/media/embed/reader/pinboard.png') no-repeat 0 0;
} }
.NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-buffer {
background: transparent url('/media/embed/reader/buffer.png') no-repeat 0 0;
background-size: 16px;
}
.NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-diigo { .NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-diigo {
background: transparent url('/media/embed/reader/diigo.png') no-repeat 0 0; background: transparent url('/media/embed/reader/diigo.png') no-repeat 0 0;
} }
@ -6145,7 +6149,8 @@ form.opml_import_form input {
background-size: 16px; background-size: 16px;
} }
.NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-twitter { .NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-twitter {
background: transparent url('/media/embed/reader/twitter_icon.png') no-repeat 0 0; background: transparent url('/media/embed/reader/twitter_bird.png') no-repeat 0 0;
background-size: 16px;
} }
.NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-facebook { .NB-menu-manage .NB-menu-manage-story-thirdparty .NB-menu-manage-thirdparty-facebook {
background: transparent url('/media/embed/reader/facebook_icon.png') no-repeat 0 0; background: transparent url('/media/embed/reader/facebook_icon.png') no-repeat 0 0;
@ -6193,6 +6198,13 @@ form.opml_import_form input {
.NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-pinboard .NB-menu-manage-thirdparty-pinboard { .NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-pinboard .NB-menu-manage-thirdparty-pinboard {
opacity: 1; opacity: 1;
} }
.NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-buffer .NB-menu-manage-image,
.NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-buffer .NB-menu-manage-thirdparty-icon {
opacity: .2;
}
.NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-buffer .NB-menu-manage-thirdparty-buffer {
opacity: 1;
}
.NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-diigo .NB-menu-manage-image, .NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-diigo .NB-menu-manage-image,
.NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-diigo .NB-menu-manage-thirdparty-icon { .NB-menu-manage .NB-menu-manage-story-thirdparty.NB-menu-manage-highlight-diigo .NB-menu-manage-thirdparty-icon {
opacity: .2; opacity: .2;
@ -8359,7 +8371,8 @@ form.opml_import_form input {
margin: 0 0 0 2px; margin: 0 0 0 2px;
} }
.NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-twitter] { .NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-twitter] {
background: transparent url('/media/embed/reader/twitter.png') no-repeat 0 0; background: transparent url('/media/embed/reader/twitter_bird.png') no-repeat 0 0;
background-size: 16px;
} }
.NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-facebook] { .NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-facebook] {
background: transparent url('/media/embed/reader/facebook.gif') no-repeat 0 0; background: transparent url('/media/embed/reader/facebook.gif') no-repeat 0 0;
@ -8376,6 +8389,10 @@ form.opml_import_form input {
.NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-pinboard] { .NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-pinboard] {
background: transparent url('/media/embed/reader/pinboard.png') no-repeat 0 0; background: transparent url('/media/embed/reader/pinboard.png') no-repeat 0 0;
} }
.NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-buffer] {
background: transparent url('/media/embed/reader/buffer.png') no-repeat 0 0;
background-size: 16px;
}
.NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-diigo] { .NB-modal-preferences .NB-preference-story-share label[for=NB-preference-story-share-diigo] {
background: transparent url('/media/embed/reader/diigo.png') no-repeat 0 0; background: transparent url('/media/embed/reader/diigo.png') no-repeat 0 0;
} }

BIN
media/img/reader/buffer.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.4 KiB

View file

@ -2096,6 +2096,20 @@
NEWSBLUR.assets.stories.mark_read(story, {skip_delay: true}); NEWSBLUR.assets.stories.mark_read(story, {skip_delay: true});
}, },
send_story_to_buffer: function(story_id) {
var story = this.model.get_story(story_id);
var url = 'https://bufferapp.com/add?source=newsblur&';
var buffer_url = [
url,
'url=',
encodeURIComponent(story.get('story_permalink')),
'&text=',
encodeURIComponent(story.get('story_title'))
].join('');
window.open(buffer_url, '_blank');
NEWSBLUR.assets.stories.mark_read(story, {skip_delay: true});
},
send_story_to_diigo: function(story_id) { send_story_to_diigo: function(story_id) {
var story = this.model.get_story(story_id); var story = this.model.get_story(story_id);
var url = 'http://www.diigo.com/post?'; var url = 'http://www.diigo.com/post?';
@ -3008,6 +3022,11 @@
}, this)).bind('mouseleave', _.bind(function(e) { }, this)).bind('mouseleave', _.bind(function(e) {
$(e.target).siblings('.NB-menu-manage-title').text('Email story').parent().removeClass('NB-menu-manage-highlight-pinboard'); $(e.target).siblings('.NB-menu-manage-title').text('Email story').parent().removeClass('NB-menu-manage-highlight-pinboard');
}, this))), }, this))),
(NEWSBLUR.Preferences['story_share_buffer'] && $.make('div', { className: 'NB-menu-manage-thirdparty-icon NB-menu-manage-thirdparty-buffer'}).bind('mouseenter', _.bind(function(e) {
$(e.target).siblings('.NB-menu-manage-title').text('Buffer').parent().addClass('NB-menu-manage-highlight-buffer');
}, this)).bind('mouseleave', _.bind(function(e) {
$(e.target).siblings('.NB-menu-manage-title').text('Email story').parent().removeClass('NB-menu-manage-highlight-buffer');
}, this))),
(NEWSBLUR.Preferences['story_share_diigo'] && $.make('div', { className: 'NB-menu-manage-thirdparty-icon NB-menu-manage-thirdparty-diigo'}).bind('mouseenter', _.bind(function(e) { (NEWSBLUR.Preferences['story_share_diigo'] && $.make('div', { className: 'NB-menu-manage-thirdparty-icon NB-menu-manage-thirdparty-diigo'}).bind('mouseenter', _.bind(function(e) {
$(e.target).siblings('.NB-menu-manage-title').text('Diigo').parent().addClass('NB-menu-manage-highlight-diigo'); $(e.target).siblings('.NB-menu-manage-title').text('Diigo').parent().addClass('NB-menu-manage-highlight-diigo');
}, this)).bind('mouseleave', _.bind(function(e) { }, this)).bind('mouseleave', _.bind(function(e) {
@ -3058,6 +3077,8 @@
this.send_story_to_readability(story.id); this.send_story_to_readability(story.id);
} else if ($target.hasClass('NB-menu-manage-thirdparty-pinboard')) { } else if ($target.hasClass('NB-menu-manage-thirdparty-pinboard')) {
this.send_story_to_pinboard(story.id); this.send_story_to_pinboard(story.id);
} else if ($target.hasClass('NB-menu-manage-thirdparty-buffer')) {
this.send_story_to_buffer(story.id);
} else if ($target.hasClass('NB-menu-manage-thirdparty-diigo')) { } else if ($target.hasClass('NB-menu-manage-thirdparty-diigo')) {
this.send_story_to_diigo(story.id); this.send_story_to_diigo(story.id);
} else if ($target.hasClass('NB-menu-manage-thirdparty-kippt')) { } else if ($target.hasClass('NB-menu-manage-thirdparty-kippt')) {

View file

@ -473,6 +473,10 @@ _.extend(NEWSBLUR.ReaderPreferences.prototype, {
$.make('input', { type: 'checkbox', id: 'NB-preference-story-share-pinboard', name: 'story_share_pinboard' }), $.make('input', { type: 'checkbox', id: 'NB-preference-story-share-pinboard', name: 'story_share_pinboard' }),
$.make('label', { 'for': 'NB-preference-story-share-pinboard' }) $.make('label', { 'for': 'NB-preference-story-share-pinboard' })
]), ]),
$.make('div', { className: 'NB-preference-option', title: 'Buffer' }, [
$.make('input', { type: 'checkbox', id: 'NB-preference-story-share-buffer', name: 'story_share_buffer' }),
$.make('label', { 'for': 'NB-preference-story-share-buffer' })
]),
$.make('div', { className: 'NB-preference-option', title: 'Diigo' }, [ $.make('div', { className: 'NB-preference-option', title: 'Diigo' }, [
$.make('input', { type: 'checkbox', id: 'NB-preference-story-share-diigo', name: 'story_share_diigo' }), $.make('input', { type: 'checkbox', id: 'NB-preference-story-share-diigo', name: 'story_share_diigo' }),
$.make('label', { 'for': 'NB-preference-story-share-diigo' }) $.make('label', { 'for': 'NB-preference-story-share-diigo' })

View file

@ -2,7 +2,7 @@ fs = require 'fs'
redis = require 'redis' redis = require 'redis'
log = require './log.js' log = require './log.js'
REDIS_SERVER = if process.env.NODE_ENV == 'development' then 'localhost' else 'db13' REDIS_SERVER = if process.env.NODE_ENV == 'development' then 'localhost' else 'db_redis_pubsub'
SECURE = !!process.env.NODE_SSL SECURE = !!process.env.NODE_SSL
# client = redis.createClient 6379, REDIS_SERVER # client = redis.createClient 6379, REDIS_SERVER
@ -49,10 +49,16 @@ io.sockets.on 'connection', (socket) ->
if not @username if not @username
return return
socket.on "error", (err) ->
console.log " ---> Error (socket): #{err}"
socket.subscribe?.end() socket.subscribe?.end()
socket.subscribe = redis.createClient 6379, REDIS_SERVER socket.subscribe = redis.createClient 6379, REDIS_SERVER
socket.subscribe.subscribe @feeds socket.subscribe.on "error", (err) ->
socket.subscribe.subscribe @username console.log " ---> Error: #{err}"
socket.subscribe.end()
socket.subscribe.on "connect", =>
socket.subscribe.subscribe @feeds
socket.subscribe.subscribe @username
socket.subscribe.on 'message', (channel, message) => socket.subscribe.on 'message', (channel, message) =>
log.info @username, "Update on #{channel}: #{message}" log.info @username, "Update on #{channel}: #{message}"
@ -66,3 +72,6 @@ io.sockets.on 'connection', (socket) ->
log.info @username, "Disconnect (#{@feeds?.length} feeds, #{ip})," + log.info @username, "Disconnect (#{@feeds?.length} feeds, #{ip})," +
" there are now #{io.sockets.clients().length-1} users. " + " there are now #{io.sockets.clients().length-1} users. " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}" " #{if SECURE then "(SSL)" else "(non-SSL)"}"
io.sockets.on 'error', (err) ->
console.log " ---> Error (sockets): #{err}"

View file

@ -8,7 +8,7 @@
log = require('./log.js'); log = require('./log.js');
REDIS_SERVER = process.env.NODE_ENV === 'development' ? 'localhost' : 'db13'; REDIS_SERVER = process.env.NODE_ENV === 'development' ? 'localhost' : 'db_redis_pubsub';
SECURE = !!process.env.NODE_SSL; SECURE = !!process.env.NODE_SSL;
@ -48,12 +48,21 @@
if (!this.username) { if (!this.username) {
return; return;
} }
socket.on("error", function(err) {
return console.log(" ---> Error (socket): " + err);
});
if ((_ref = socket.subscribe) != null) { if ((_ref = socket.subscribe) != null) {
_ref.end(); _ref.end();
} }
socket.subscribe = redis.createClient(6379, REDIS_SERVER); socket.subscribe = redis.createClient(6379, REDIS_SERVER);
socket.subscribe.subscribe(this.feeds); socket.subscribe.on("error", function(err) {
socket.subscribe.subscribe(this.username); console.log(" ---> Error: " + err);
return socket.subscribe.end();
});
socket.subscribe.on("connect", function() {
socket.subscribe.subscribe(_this.feeds);
return socket.subscribe.subscribe(_this.username);
});
return socket.subscribe.on('message', function(channel, message) { return socket.subscribe.on('message', function(channel, message) {
log.info(_this.username, "Update on " + channel + ": " + message); log.info(_this.username, "Update on " + channel + ": " + message);
if (channel === _this.username) { if (channel === _this.username) {
@ -72,4 +81,8 @@
}); });
}); });
io.sockets.on('error', function(err) {
return console.log(" ---> Error (sockets): " + err);
});
}).call(this); }).call(this);

View file

@ -66,6 +66,7 @@ SITE_ID = 1
USE_I18N = False USE_I18N = False
LOGIN_REDIRECT_URL = '/' LOGIN_REDIRECT_URL = '/'
LOGIN_URL = '/reader/login' LOGIN_URL = '/reader/login'
MEDIA_URL = '/media/'
# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash. # trailing slash.
# Examples: "http://foo.com/media/", "/media/". # Examples: "http://foo.com/media/", "/media/".
@ -199,7 +200,7 @@ INTERNAL_IPS = ('127.0.0.1',)
LOGGING_LOG_SQL = True LOGGING_LOG_SQL = True
APPEND_SLASH = False APPEND_SLASH = False
SOUTH_TESTS_MIGRATE = False SOUTH_TESTS_MIGRATE = False
SESSION_ENGINE = "django.contrib.sessions.backends.db" SESSION_ENGINE = 'redis_sessions.session'
TEST_RUNNER = "utils.testrunner.TestRunner" TEST_RUNNER = "utils.testrunner.TestRunner"
SESSION_COOKIE_NAME = 'newsblur_sessionid' SESSION_COOKIE_NAME = 'newsblur_sessionid'
SESSION_COOKIE_AGE = 60*60*24*365*2 # 2 years SESSION_COOKIE_AGE = 60*60*24*365*2 # 2 years
@ -380,7 +381,7 @@ CELERYBEAT_SCHEDULE = {
}, },
'activate-next-new-user': { 'activate-next-new-user': {
'task': 'activate-next-new-user', 'task': 'activate-next-new-user',
'schedule': datetime.timedelta(minutes=5), 'schedule': datetime.timedelta(minutes=3.5),
'options': {'queue': 'beat_tasks'}, 'options': {'queue': 'beat_tasks'},
}, },
} }
@ -390,11 +391,11 @@ CELERYBEAT_SCHEDULE = {
# ========= # =========
MONGO_DB = { MONGO_DB = {
'host': '127.0.0.1:27017', 'host': 'db_mongo:27017',
'name': 'newsblur', 'name': 'newsblur',
} }
MONGO_ANALYTICS_DB = { MONGO_ANALYTICS_DB = {
'host': '127.0.0.1:27017', 'host': 'db_mongo_analytics:27017',
'name': 'nbanalytics', 'name': 'nbanalytics',
} }
@ -429,14 +430,15 @@ class MasterSlaveRouter(object):
# ========= # =========
REDIS = { REDIS = {
'host': 'db12', 'host': 'db_redis',
} }
REDIS2 = { REDIS_PUBSUB = {
'host': 'db13', 'host': 'db_redis_pubsub',
} }
REDIS3 = { REDIS_STORY = {
'host': 'db11', 'host': 'db_redis_story',
} }
CELERY_REDIS_DB = 4 CELERY_REDIS_DB = 4
SESSION_REDIS_DB = 5 SESSION_REDIS_DB = 5
@ -444,7 +446,7 @@ SESSION_REDIS_DB = 5
# = Elasticsearch = # = Elasticsearch =
# ================= # =================
ELASTICSEARCH_HOSTS = ['db01:9200'] ELASTICSEARCH_HOSTS = ['db_search:9200']
# =============== # ===============
# = Social APIs = # = Social APIs =
@ -460,7 +462,7 @@ TWITTER_CONSUMER_SECRET = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
# = AWS Backing = # = AWS Backing =
# =============== # ===============
ORIGINAL_PAGE_SERVER = "db01:3060" ORIGINAL_PAGE_SERVER = "db_pages:3060"
BACKED_BY_AWS = { BACKED_BY_AWS = {
'pages_on_s3': False, 'pages_on_s3': False,
@ -533,6 +535,18 @@ else:
BROKER_BACKEND = "redis" BROKER_BACKEND = "redis"
BROKER_URL = "redis://%s:6379/%s" % (REDIS['host'], CELERY_REDIS_DB) BROKER_URL = "redis://%s:6379/%s" % (REDIS['host'], CELERY_REDIS_DB)
CELERY_RESULT_BACKEND = BROKER_URL CELERY_RESULT_BACKEND = BROKER_URL
SESSION_REDIS_HOST = REDIS['host']
CACHES = {
'default': {
'BACKEND': 'redis_cache.RedisCache',
'LOCATION': '%s:6379' % REDIS['host'],
'OPTIONS': {
'DB': 6,
'PARSER_CLASS': 'redis.connection.HiredisParser'
},
},
}
# ========= # =========
# = Mongo = # = Mongo =
@ -540,7 +554,7 @@ CELERY_RESULT_BACKEND = BROKER_URL
MONGO_DB_DEFAULTS = { MONGO_DB_DEFAULTS = {
'name': 'newsblur', 'name': 'newsblur',
'host': 'db02:27017', 'host': 'db_mongo:27017',
'alias': 'default', 'alias': 'default',
} }
MONGO_DB = dict(MONGO_DB_DEFAULTS, **MONGO_DB) MONGO_DB = dict(MONGO_DB_DEFAULTS, **MONGO_DB)
@ -555,7 +569,7 @@ MONGODB = connect(MONGO_DB.pop('name'), **MONGO_DB)
MONGO_ANALYTICS_DB_DEFAULTS = { MONGO_ANALYTICS_DB_DEFAULTS = {
'name': 'nbanalytics', 'name': 'nbanalytics',
'host': 'db30:27017', 'host': 'db_mongo_analytics:27017',
'alias': 'nbanalytics', 'alias': 'nbanalytics',
} }
MONGO_ANALYTICS_DB = dict(MONGO_ANALYTICS_DB_DEFAULTS, **MONGO_ANALYTICS_DB) MONGO_ANALYTICS_DB = dict(MONGO_ANALYTICS_DB_DEFAULTS, **MONGO_ANALYTICS_DB)
@ -572,11 +586,11 @@ REDIS_STATISTICS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=3
REDIS_FEED_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=4) REDIS_FEED_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=4)
REDIS_SESSION_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=5) REDIS_SESSION_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=5)
# REDIS_CACHE_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=6) # Duped in CACHES # REDIS_CACHE_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=6) # Duped in CACHES
REDIS_STORY_HASH_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=8) REDIS_STORY_HASH_POOL = redis.ConnectionPool(host=REDIS_STORY['host'], port=6379, db=1)
REDIS_PUBSUB_POOL = redis.ConnectionPool(host=REDIS2['host'], port=6379, db=0) REDIS_PUBSUB_POOL = redis.ConnectionPool(host=REDIS_PUBSUB['host'], port=6379, db=0)
REDIS_STORY_HASH_POOL2 = redis.ConnectionPool(host=REDIS3['host'], port=6379, db=1) REDIS_STORY_HASH_POOL2 = redis.ConnectionPool(host=REDIS['host'], port=6379, db=8)
# ========== # ==========
# = Assets = # = Assets =

View file

@ -0,0 +1,16 @@
import os
import sys
CURRENT_DIR = os.path.dirname(__file__)
NEWSBLUR_DIR = ''.join([CURRENT_DIR, '/../../'])
sys.path.insert(0, NEWSBLUR_DIR)
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
import time
import s3
from django.conf import settings
filename = 'backup_redis_story_%s.rdb.gz' % time.strftime('%Y-%m-%d-%H-%M')
path = '/var/lib/redis/dump.rdb'
print 'Uploading %s (from %s) to S3...' % (filename, path)
s3.save_file_in_s3(path, name=filename)