Moving task feeds to its own beat queue.

apps/rss_feeds/tasks.py

@@ -19,10 +19,18 @@ class TaskFeeds(Task):
         feeds = Feed.objects.filter(
             next_scheduled_update__lte=now,
             active=True,
-            active_premium_subscribers__gte=0
+            active_premium_subscribers__gte=10
         ).order_by('?')
         active_count = feeds.count()
         
+        # Regular feeds
+        feeds = Feed.objects.filter(
+            next_scheduled_update__lte=now,
+            active=True,
+            active_premium_subscribers__gte=0
+        ).order_by('?')[:1000]
+        active_count = feeds.count()
+        
         # Mistakenly inactive feeds
         day = now - datetime.timedelta(days=1)
         inactive_feeds = Feed.objects.filter(

config/pgbouncer.conf

@@ -2,7 +2,7 @@
 [databases]
 
 newsblur = host=db01 port=5432 dbname=newsblur
-# slave    = host=db02 port=5432 dbname=newsblur
+slave    = host=db10 port=5432 dbname=newsblur
 
 ;; Configuation section
 [pgbouncer]

config/supervisor_celeryd_beat.conf

@@ -1,5 +1,5 @@
 [program:celeryd_beat]
-command=/srv/newsblur/manage.py celeryd --loglevel=INFO -Q beat_tasks -c 1
+command=/srv/newsblur/manage.py celeryd --loglevel=INFO -Q beat_tasks -c 3
 directory=/srv/newsblur
 user=sclay
 numprocs=1

config/supervisor_celeryd_beat_feeds.conf

@@ -0,0 +1,19 @@
+[program:celeryd_beat]
+command=/srv/newsblur/manage.py celeryd --loglevel=INFO -Q beat_feeds_task -c 1
+directory=/srv/newsblur
+user=sclay
+numprocs=1
+stdout_logfile=/var/log/celeryd.log
+stderr_logfile=/var/log/celeryd.log
+autostart=true
+autorestart=true
+startsecs=10
+;process_name=%(program_name)s_%(process_num)03d
+
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 60
+
+; if rabbitmq is supervised, set its priority higher
+; so it starts first
+priority=998

fabfile.py

@@ -799,6 +799,7 @@ def enable_celerybeat():
         run('mkdir -p data')
     put('config/supervisor_celerybeat.conf', '/etc/supervisor/conf.d/celerybeat.conf', use_sudo=True)
     put('config/supervisor_celeryd_beat.conf', '/etc/supervisor/conf.d/celeryd_beat.conf', use_sudo=True)
+    put('config/supervisor_celeryd_beat_feeds.conf', '/etc/supervisor/conf.d/celeryd_beat_feeds.conf', use_sudo=True)
     sudo('supervisorctl reread')
     sudo('supervisorctl update')
     

settings.py

@@ -312,6 +312,11 @@ CELERY_QUEUES = {
         "exchange_type": "direct",
         "binding_key": "beat_tasks"
     },
+    "beat_feeds_task": {
+        "exchange": "beat_feeds_task",
+        "exchange_type": "direct",
+        "binding_key": "beat_feeds_task"
+    },
 }
 CELERY_DEFAULT_QUEUE = "work_queue"
 
@@ -330,14 +335,14 @@ CELERY_DISABLE_RATE_LIMITS  = True
 SECONDS_TO_DELAY_CELERY_EMAILS = 60
 
 CELERYBEAT_SCHEDULE = {
-    'freshen-homepage': {
-        'task': 'freshen-homepage',
-        'schedule': datetime.timedelta(hours=1),
-        'options': {'queue': 'beat_tasks'},
-    },
     'task-feeds': {
         'task': 'task-feeds',
         'schedule': datetime.timedelta(minutes=1),
+        'options': {'queue': 'beat_feeds_task'},
+    },
+    'freshen-homepage': {
+        'task': 'freshen-homepage',
+        'schedule': datetime.timedelta(hours=1),
         'options': {'queue': 'beat_tasks'},
     },
     'collect-stats': {