From 9052636418ea05cbc1de296f88843b69fb00f5c5 Mon Sep 17 00:00:00 2001
From: Samuel Clay <samuel@ofbrooklyn.com>
Date: Thu, 14 Mar 2013 12:55:10 -0700
Subject: [PATCH] Moving task feeds to its own beat queue.

---
 apps/rss_feeds/tasks.py                   | 10 +++++++++-
 config/pgbouncer.conf                     |  2 +-
 config/supervisor_celeryd_beat.conf       |  2 +-
 config/supervisor_celeryd_beat_feeds.conf | 19 +++++++++++++++++++
 fabfile.py                                |  1 +
 settings.py                               | 15 ++++++++++-----
 6 files changed, 41 insertions(+), 8 deletions(-)
 create mode 100644 config/supervisor_celeryd_beat_feeds.conf

diff --git a/apps/rss_feeds/tasks.py b/apps/rss_feeds/tasks.py
index 92fcb1f75..b93d968f8 100644
--- a/apps/rss_feeds/tasks.py
+++ b/apps/rss_feeds/tasks.py
@@ -19,10 +19,18 @@ class TaskFeeds(Task):
         feeds = Feed.objects.filter(
             next_scheduled_update__lte=now,
             active=True,
-            active_premium_subscribers__gte=0
+            active_premium_subscribers__gte=10
         ).order_by('?')
         active_count = feeds.count()
         
+        # Regular feeds
+        feeds = Feed.objects.filter(
+            next_scheduled_update__lte=now,
+            active=True,
+            active_premium_subscribers__gte=0
+        ).order_by('?')[:1000]
+        active_count = feeds.count()
+        
         # Mistakenly inactive feeds
         day = now - datetime.timedelta(days=1)
         inactive_feeds = Feed.objects.filter(
diff --git a/config/pgbouncer.conf b/config/pgbouncer.conf
index 015353529..a2eb8c0d3 100644
--- a/config/pgbouncer.conf
+++ b/config/pgbouncer.conf
@@ -2,7 +2,7 @@
 [databases]
 
 newsblur = host=db01 port=5432 dbname=newsblur
-# slave    = host=db02 port=5432 dbname=newsblur
+slave    = host=db10 port=5432 dbname=newsblur
 
 ;; Configuation section
 [pgbouncer]
diff --git a/config/supervisor_celeryd_beat.conf b/config/supervisor_celeryd_beat.conf
index 13dcb04b5..a0ca2b79e 100644
--- a/config/supervisor_celeryd_beat.conf
+++ b/config/supervisor_celeryd_beat.conf
@@ -1,5 +1,5 @@
 [program:celeryd_beat]
-command=/srv/newsblur/manage.py celeryd --loglevel=INFO -Q beat_tasks -c 1
+command=/srv/newsblur/manage.py celeryd --loglevel=INFO -Q beat_tasks -c 3
 directory=/srv/newsblur
 user=sclay
 numprocs=1
diff --git a/config/supervisor_celeryd_beat_feeds.conf b/config/supervisor_celeryd_beat_feeds.conf
new file mode 100644
index 000000000..55929c30e
--- /dev/null
+++ b/config/supervisor_celeryd_beat_feeds.conf
@@ -0,0 +1,19 @@
+[program:celeryd_beat]
+command=/srv/newsblur/manage.py celeryd --loglevel=INFO -Q beat_feeds_task -c 1
+directory=/srv/newsblur
+user=sclay
+numprocs=1
+stdout_logfile=/var/log/celeryd.log
+stderr_logfile=/var/log/celeryd.log
+autostart=true
+autorestart=true
+startsecs=10
+;process_name=%(program_name)s_%(process_num)03d
+
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 60
+
+; if rabbitmq is supervised, set its priority higher
+; so it starts first
+priority=998
diff --git a/fabfile.py b/fabfile.py
index f8a896584..833047cbf 100644
--- a/fabfile.py
+++ b/fabfile.py
@@ -799,6 +799,7 @@ def enable_celerybeat():
         run('mkdir -p data')
     put('config/supervisor_celerybeat.conf', '/etc/supervisor/conf.d/celerybeat.conf', use_sudo=True)
     put('config/supervisor_celeryd_beat.conf', '/etc/supervisor/conf.d/celeryd_beat.conf', use_sudo=True)
+    put('config/supervisor_celeryd_beat_feeds.conf', '/etc/supervisor/conf.d/celeryd_beat_feeds.conf', use_sudo=True)
     sudo('supervisorctl reread')
     sudo('supervisorctl update')
     
diff --git a/settings.py b/settings.py
index 7182c2600..8786d68ce 100644
--- a/settings.py
+++ b/settings.py
@@ -312,6 +312,11 @@ CELERY_QUEUES = {
         "exchange_type": "direct",
         "binding_key": "beat_tasks"
     },
+    "beat_feeds_task": {
+        "exchange": "beat_feeds_task",
+        "exchange_type": "direct",
+        "binding_key": "beat_feeds_task"
+    },
 }
 CELERY_DEFAULT_QUEUE = "work_queue"
 
@@ -330,14 +335,14 @@ CELERY_DISABLE_RATE_LIMITS  = True
 SECONDS_TO_DELAY_CELERY_EMAILS = 60
 
 CELERYBEAT_SCHEDULE = {
-    'freshen-homepage': {
-        'task': 'freshen-homepage',
-        'schedule': datetime.timedelta(hours=1),
-        'options': {'queue': 'beat_tasks'},
-    },
     'task-feeds': {
         'task': 'task-feeds',
         'schedule': datetime.timedelta(minutes=1),
+        'options': {'queue': 'beat_feeds_task'},
+    },
+    'freshen-homepage': {
+        'task': 'freshen-homepage',
+        'schedule': datetime.timedelta(hours=1),
         'options': {'queue': 'beat_tasks'},
     },
     'collect-stats': {