public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-09-18 21:43:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adding logging, feed cleanup to forced fetch feeds, and expirations on user read stories.

Browse source
This commit is contained in:
Samuel Clay 2013-04-07 17:19:59 -07:00
parent 0e7bcab43d
commit 48c5d4d53e
9 changed files with 25 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
apps/reader/models.py
View file

@ -667,6 +667,7 @@ class MUserStory(mongo.Document):
if self.story_db_id:
all_read_stories_key = 'RS:%s' % (self.user_id)
r.sadd(all_read_stories_key, self.story_db_id)
r.expire(all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
read_story_key = 'RS:%s:%s' % (self.user_id, self.feed_id)
r.sadd(read_story_key, self.story_db_id)

4
apps/rss_feeds/models.py
View file

@ -773,8 +773,6 @@ class Feed(models.Model):
feed.last_update = datetime.datetime.utcnow()
feed.set_next_scheduled_update()
r.zadd('fetched_feeds_last_hour', feed.pk, int(datetime.datetime.now().strftime('%s')))
if options['force']:
feed.sync_redis()
if not feed or original_feed_id != feed.pk:
logging.info(" ---> ~FRFeed changed id, removing %s from tasked_feeds queue..." % original_feed_id)
@ -1574,6 +1572,8 @@ class MStory(mongo.Document):
if self.id and self.story_date > DAYS_OF_UNREAD:
r.sadd('F:%s' % self.story_feed_id, self.id)
r.zadd('zF:%s' % self.story_feed_id, self.id, time.mktime(self.story_date.timetuple()))
r.expire('F:%s' % self.story_feed_id, settings.DAYS_OF_UNREAD*24*60*60)
r.expire('zF:%s' % self.story_feed_id, settings.DAYS_OF_UNREAD*24*60*60)
def remove_from_redis(self, r=None):
if not r:

5
apps/rss_feeds/tasks.py
View file

@ -26,8 +26,9 @@ class TaskFeeds(Task):
queued_feeds = r.zrangebyscore('scheduled_updates', 0, now_timestamp)
r.zremrangebyscore('scheduled_updates', 0, now_timestamp)
r.sadd('queued_feeds', *queued_feeds)
logging.debug(" ---> ~SN~FBQueuing ~SB%s~SN stale feeds (~SB%s~SN/%s queued/scheduled)" % (
logging.debug(" ---> ~SN~FBQueuing ~SB%s~SN stale feeds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
len(queued_feeds),
r.zcard('tasked_feeds'),
r.scard('queued_feeds'),
r.zcard('scheduled_updates')))
@ -85,7 +86,7 @@ class TaskFeeds(Task):
Feed.task_feeds(refresh_feeds, verbose=False)
Feed.task_feeds(old_feeds, verbose=False)
logging.debug(" ---> ~SN~FBTasking took ~SB%s~SN seconds (~SB%s~SN/~SB%s~SN/%s tasked/queued/scheduled)" % (
logging.debug(" ---> ~SN~FBTasking took ~SB%s~SN seconds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
int((time.time() - start)),
r.zcard('tasked_feeds'),
r.scard('queued_feeds'),

2
config/redis.conf
View file

@ -555,5 +555,5 @@ client-output-buffer-limit pubsub 32mb 8mb 60
# to customize a few per-server settings. Include files can include
# other files, so use this wisely.
#
# include /path/to/local.conf
include /etc/redis_server.conf
# include /path/to/other.conf

0
config/redis_master.conf Normal file
View file

1
config/redis_slave.conf Normal file
View file

@ -0,0 +1 @@
slaveof db10 6379

18
fabfile.py vendored
View file

@ -337,6 +337,8 @@ def setup_db(engine=None, skip_common=False):
setup_mongo()
elif engine == "redis":
setup_redis()
elif engine == "redis_slave":
setup_redis(slave=True)
setup_gunicorn(supervisor=False)
setup_db_munin()
@ -456,10 +458,10 @@ def setup_imaging():
def setup_supervisor():
sudo('apt-get -y install supervisor')
put('config/supervisord.conf', '/etc/supervisor/supervisord.conf')
sudo('/etc/init.d/supervisord stop')
put('config/supervisord.conf', '/etc/supervisor/supervisord.conf', use_sudo=True)
sudo('/etc/init.d/supervisor stop')
sudo('sleep 2')
sudo('/etc/init.d/supervisord start')
sudo('/etc/init.d/supervisor start')
# @parallel
def setup_hosts():
@ -563,7 +565,7 @@ def setup_sudoers(user=None):
sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % (user or env.user))
def setup_nginx():
NGINX_VERSION = '1.2.2'
NGINX_VERSION = '1.2.8'
with cd(env.VENDOR_PATH), settings(warn_only=True):
sudo("groupadd nginx")
sudo("useradd -g nginx -d /var/www/htdocs -s /bin/false nginx")
@ -834,8 +836,8 @@ def setup_mongo_mms():
sudo('supervisorctl update')
def setup_redis():
redis_version = '2.6.11'
def setup_redis(slave=False):
redis_version = '2.6.12'
with cd(env.VENDOR_PATH):
run('wget http://redis.googlecode.com/files/redis-%s.tar.gz' % redis_version)
run('tar -xzf redis-%s.tar.gz' % redis_version)
@ -845,6 +847,10 @@ def setup_redis():
put('config/redis-init', '/etc/init.d/redis', use_sudo=True)
sudo('chmod u+x /etc/init.d/redis')
put('config/redis.conf', '/etc/redis.conf', use_sudo=True)
if slave:
put('config/redis_slave.conf', '/etc/redis_server.conf', use_sudo=True)
else:
put('config/redis_master.conf', '/etc/redis_server.conf', use_sudo=True)
sudo('mkdir -p /var/lib/redis')
sudo('update-rc.d redis defaults')
sudo('/etc/init.d/redis stop')

2
templates/maintenance_off.html
View file

@ -77,7 +77,7 @@
<img src="/media/img/logo_512.png" class="logo">
<h1>NewsBlur is in <span class="error404">maintenance mode</span></h1>
<div class="description">
<p>This will take approximately 5 minutes. I'm upgrading PostgreSQL and re-building indexes for optimum performance. Tonight's a rough night, but hopefully this will be the last of it.</p>
<p>This will take approximately 5-10 minutes. I'm upgrading Redis, which is throwing faults due to memory issues.</p>
<p>To pass the time, go surf <a href="http://mlkshk.com/popular">MLKSHK's popular page</a>.</p>
<p></p>
</div>

5
utils/feed_fetcher.py
View file

@ -369,10 +369,13 @@ class Dispatcher:
feed.known_good = True
feed.fetched_once = True
feed = feed.save()
if random.random() <= 0.02:
if self.options['force'] or random.random() <= 0.02:
logging.debug(' ---> [%-30s] ~FBPerforming feed cleanup...' % (feed.title[:30],))
start_cleanup = time.time()
feed.sync_redis()
MUserStory.delete_old_stories(feed_id=feed.pk)
MUserStory.sync_all_redis(feed_id=feed.pk)
logging.debug(' ---> [%-30s] ~FBDone with feed cleanup. Took %s sec.' % (feed.title[:30], time.time() - start_cleanup))
try:
self.count_unreads_for_subscribers(feed)
except TimeoutError: