Handling too many feeds tasked.

ansible/roles/celery_task/tasks/main.yml

@@ -43,7 +43,7 @@
       max-size: 100m
     healthcheck: 
       # test: celery inspect ping -A newsblur_web -d celery@$HOSTNAME
-      test: bash -c "(($(date +%s) - $(stat /srv/newsblur/logs/newsblur.log -c %Y) < 120)) && exit 0 || exit 1"
+      test: bash -c "(($(date +%s) - $(stat /srv/newsblur/logs/newsblur.log -c %Y) < 600)) && exit 0 || exit 1"
       interval: 60s
       timeout: 10s
       retries: 3

ansible/roles/postgres/tasks/main.yml

@@ -105,7 +105,7 @@
       - /srv/newsblur/docker/postgres/postgres_ident-13.conf:/etc/postgresql/pg_ident.conf
       - /home/nb/.ssh/id_rsa:/var/lib/postgresql/.ssh/id_rsa
     restart_policy: unless-stopped
-  when: (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ['db-postgres-primary', 'db-postgres', 'hdb-postgres']
+  when: (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ['db-postgres-primary', 'db-postgres', 'hdb-postgres', 'hdb-postgres-secondary']
 
 # - name: Change ownership in postgres docker container
 #   become: yes

ansible/roles/ufw/tasks/main.yml

@@ -6,6 +6,7 @@
     do_hosts: "{{ groups['NewsBlur_Docker'] | map('extract', hostvars, ['ansible_host']) }}"
     
 - name: Generate UFW batch script
+  become: yes
   template:
     src: ufw_rules.sh.j2
     dest: /tmp/ufw_rules.sh

apps/rss_feeds/tasks.py

@@ -2,13 +2,16 @@ import datetime
 import os
 import shutil
 import time
+
 import redis
-from newsblur_web.celeryapp import app
 from celery.exceptions import SoftTimeLimitExceeded
-from utils import log as logging
 from django.conf import settings
+
 from apps.profile.middleware import DBProfilerMiddleware
+from newsblur_web.celeryapp import app
+from utils import log as logging
 from utils.redis_raw_log_middleware import RedisDumpMiddleware
+
 FEED_TASKING_MAX = 10000
 
 @app.task(name='task-feeds')
@@ -45,6 +48,7 @@ def TaskFeeds():
     else:
         logging.debug(" ---> ~SN~FBToo many tasked feeds. ~SB%s~SN tasked." % tasked_feeds_size)
         active_count = 0
+        feeds = []
     
     logging.debug(" ---> ~SN~FBTasking %s feeds took ~SB%s~SN seconds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
                     active_count,

docker/haproxy/haproxy.consul.cfg.j2

@@ -204,7 +204,7 @@ backend db_metrics
 backend metrics
     option httpchk GET /_haproxychk
     http-check expect rstatus 200|301
-    server app-django1 app-django1.node.nyc1.consul:8000 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    server happ-web-01 happ-web-01.node.nyc1.consul:8000 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
 
 backend postgres
     option httpchk GET /db_check/postgres
@@ -254,7 +254,7 @@ backend db_redis_sessions
 backend db_redis_pubsub
     option httpchk GET /db_check/redis_pubsub
     default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
-    server hdb-redis-pubsub hdb-redis-pubsub.node.nyc1.consul:5579
+    server hdb-redis-pubsub db-redis-pubsub.service.nyc1.consul:5579
 
 backend db_elasticsearch
     option httpchk GET /db_check/elasticsearch