From bbd6b9d4743d5b2be42b680b9bd94cb32b319179 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 6 Jan 2021 18:07:52 -0500 Subject: [PATCH 01/10] Revert "Removing unused types key from index." This reverts commit 6b384b51a2086e0e91ec30e3a1d5db9c125d8656. --- apps/notifications/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/notifications/models.py b/apps/notifications/models.py index 91ae0081f..ac0690ec0 100644 --- a/apps/notifications/models.py +++ b/apps/notifications/models.py @@ -38,6 +38,7 @@ class MUserNotificationTokens(mongo.Document): 'collection': 'notification_tokens', 'indexes': [{'fields': ['user_id'], 'unique': True, + 'types': False, }], 'allow_inheritance': False, } From a4f1443c5c73190971c135f2765be0c90f77c619 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:43:18 -0500 Subject: [PATCH 02/10] Only warn about falling feed fetches when actually falling. --- utils/monitor_task_fetches.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index 84c565b09..561d0317a 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -18,17 +18,24 @@ def main(): admin_email = settings.ADMINS[0][1] failed = False feeds_fetched = 0 - + FETCHES_DROP_AMOUNT = 0 + monitor_key = "Monitor:task_fetches" + r = redis.Redis(connection_pool=settings.REDIS_ANALYTICS_POOL) + try: client = pymongo.MongoClient('mongodb://%s' % settings.MONGO_DB['host']) feeds_fetched = client.newsblur.statistics.find_one({"key": "feeds_fetched"})['value'] + redis_task_fetches = int(r.get(monitor_key, feeds_fetched)) except Exception, e: failed = e - if feeds_fetched < 5000000: + if feeds_fetched < 5000000 and feeds_fetched <= (redis_task_fetches - FETCHES_DROP_AMOUNT): failed = True - + if failed: + r.set(monitor_key, feeds_fetched) + r.expire(monitor_key, 60*60*3) # 3 hours + requests.post( "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME, auth=("api", settings.MAILGUN_ACCESS_KEY), From c9c891e44a2378b481742d40766297cb19d3c8c3 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:44:19 -0500 Subject: [PATCH 03/10] Needs redis --- utils/monitor_task_fetches.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index 561d0317a..b58abe744 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -7,6 +7,7 @@ import subprocess import requests from newsblur import settings import socket +import redis import pymongo def main(): @@ -35,7 +36,7 @@ def main(): if failed: r.set(monitor_key, feeds_fetched) r.expire(monitor_key, 60*60*3) # 3 hours - + requests.post( "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME, auth=("api", settings.MAILGUN_ACCESS_KEY), From 238aa3916acbc6c3c06e0c7f73bb018c3793f3af Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:45:35 -0500 Subject: [PATCH 04/10] Handling missing values --- utils/monitor_task_fetches.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index b58abe744..b1a3b455f 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -20,6 +20,7 @@ def main(): failed = False feeds_fetched = 0 FETCHES_DROP_AMOUNT = 0 + redis_task_fetches = 0 monitor_key = "Monitor:task_fetches" r = redis.Redis(connection_pool=settings.REDIS_ANALYTICS_POOL) @@ -34,9 +35,6 @@ def main(): failed = True if failed: - r.set(monitor_key, feeds_fetched) - r.expire(monitor_key, 60*60*3) # 3 hours - requests.post( "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME, auth=("api", settings.MAILGUN_ACCESS_KEY), @@ -44,6 +42,10 @@ def main(): "to": [admin_email], "subject": "%s feeds fetched falling: %s" % (hostname, feeds_fetched), "text": "Feed fetches are falling: %s" % (feeds_fetched)}) + + r.set(monitor_key, feeds_fetched) + r.expire(monitor_key, 60*60*3) # 3 hours + print(" ---> Feeds fetched falling! %s" % (feeds_fetched)) else: print(" ---> Feeds fetched OK: %s" % (feeds_fetched)) From b58ab9f9cd2543ccee3104ffb65ed7222a307ea4 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:46:52 -0500 Subject: [PATCH 05/10] Spitting out error on failure. --- utils/monitor_task_fetches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index b1a3b455f..aeed39e69 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -46,7 +46,7 @@ def main(): r.set(monitor_key, feeds_fetched) r.expire(monitor_key, 60*60*3) # 3 hours - print(" ---> Feeds fetched falling! %s" % (feeds_fetched)) + print(" ---> Feeds fetched falling! %s %s" % (feeds_fetched, failed)) else: print(" ---> Feeds fetched OK: %s" % (feeds_fetched)) From d747acc5d0a0657bff280d1a75a65e20dcde91ca Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:47:17 -0500 Subject: [PATCH 06/10] No default value --- utils/monitor_task_fetches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index aeed39e69..8cc3624e0 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -27,7 +27,7 @@ def main(): try: client = pymongo.MongoClient('mongodb://%s' % settings.MONGO_DB['host']) feeds_fetched = client.newsblur.statistics.find_one({"key": "feeds_fetched"})['value'] - redis_task_fetches = int(r.get(monitor_key, feeds_fetched)) + redis_task_fetches = int(r.get(monitor_key)) except Exception, e: failed = e From 047648b508aa6f4605cb162730bcb4aaaaa064fe Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:49:12 -0500 Subject: [PATCH 07/10] More descriptive failure email --- utils/monitor_task_fetches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index 8cc3624e0..a16e2dec4 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -40,8 +40,8 @@ def main(): auth=("api", settings.MAILGUN_ACCESS_KEY), data={"from": "NewsBlur Task Monitor: %s " % (hostname, hostname), "to": [admin_email], - "subject": "%s feeds fetched falling: %s" % (hostname, feeds_fetched), - "text": "Feed fetches are falling: %s" % (feeds_fetched)}) + "subject": "%s feeds fetched falling: %s (from %s)" % (hostname, feeds_fetched, redis_task_fetches), + "text": "Feed fetches are falling: %s (from %s)" % (feeds_fetched, redis_task_fetches)}) r.set(monitor_key, feeds_fetched) r.expire(monitor_key, 60*60*3) # 3 hours From 7d218a652ad63f6ab4c930cd16db14f6774785ba Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 20:16:25 -0500 Subject: [PATCH 08/10] Pointing to the new ssl cert. --- config/haproxy.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/haproxy.conf.template b/config/haproxy.conf.template index 18597d073..a542be20f 100644 --- a/config/haproxy.conf.template +++ b/config/haproxy.conf.template @@ -166,7 +166,7 @@ backend maintenance {{ maintenance }} listen stats - bind :1936 ssl crt newsblur.pem + bind :1936 ssl crt newsblur.com.crt stats enable stats hide-version stats realm Haproxy\ Statistics From b54c98198ea29986f53087051203f185a23f3b34 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 20 Jan 2021 13:29:12 -0500 Subject: [PATCH 09/10] Adding monitor for work queue. --- fabfile.py | 12 ++++++--- utils/monitor_work_queue.py | 54 +++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 utils/monitor_work_queue.py diff --git a/fabfile.py b/fabfile.py index 064f94252..a126dbefc 100644 --- a/fabfile.py +++ b/fabfile.py @@ -828,15 +828,14 @@ def copy_certificates(): run('ln -fs %s %s' % (privkey_path, os.path.join(cert_path, 'newsblur.com.crt.key'))) # HAProxy put(os.path.join(env.SECRETS_PATH, 'certificates/comodo/dhparams.pem'), cert_path) put(os.path.join(env.SECRETS_PATH, 'certificates/ios/aps_development.pem'), cert_path) + + # Export aps.cer from Apple issued certificate using Keychain Assistant # openssl x509 -in aps.cer -inform DER -outform PEM -out aps.pem put(os.path.join(env.SECRETS_PATH, 'certificates/ios/aps.pem'), cert_path) # Export aps.p12 from aps.cer using Keychain Assistant # openssl pkcs12 -in aps.p12 -out aps.p12.pem -nodes put(os.path.join(env.SECRETS_PATH, 'certificates/ios/aps.p12.pem'), cert_path) - # run('cat %s/newsblur.com.crt > %s/newsblur.pem' % (cert_path, cert_path)) - # run('echo "\n" >> %s/newsblur.pem' % (cert_path)) - # run('cat %s/newsblur.com.key >> %s/newsblur.pem' % (cert_path, cert_path)) - + def setup_certbot(): sudo('snap install --classic certbot') sudo('snap set certbot trust-plugin-with-root=ok') @@ -1492,6 +1491,11 @@ def setup_newsletter_monitor(): sudo('ln -fs %s/utils/monitor_newsletter_delivery.py /etc/cron.hourly/monitor_newsletter_delivery' % env.NEWSBLUR_PATH) sudo('/etc/cron.hourly/monitor_newsletter_delivery') +@parallel +def setup_queue_monitor(): + sudo('/etc/cron.hourly/monitor_work_queue') + sudo('ln -fs %s/utils/monitor_work_queue.py /etc/cron.hourly/monitor_work_queue' % env.NEWSBLUR_PATH) + @parallel def setup_redis_monitor(): run('sleep 5') # Wait for redis to startup so the log file is there diff --git a/utils/monitor_work_queue.py b/utils/monitor_work_queue.py new file mode 100644 index 000000000..0c5ba2deb --- /dev/null +++ b/utils/monitor_work_queue.py @@ -0,0 +1,54 @@ +#!/srv/newsblur/venv/newsblur/bin/python + +import sys +sys.path.append('/srv/newsblur') + +import subprocess +import requests +from newsblur import settings +import socket +import redis +import pymongo + +def main(): + df = subprocess.Popen(["df", "/"], stdout=subprocess.PIPE) + output = df.communicate()[0] + device, size, used, available, percent, mountpoint = output.split("\n")[1].split() + hostname = socket.gethostname() + percent = int(percent.strip('%')) + admin_email = settings.ADMINS[0][1] + failed = False + work_queue_size = 0 + QUEUE_DROP_AMOUNT = 0 + redis_work_queue = 0 + monitor_key = "Monitor:work_queue" + r_monitor = redis.Redis(connection_pool=settings.REDIS_ANALYTICS_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) + + try: + work_queue_size = int(r.llen("work_queue")) + redis_work_queue = int(r_monitor.get(monitor_key)) + except Exception, e: + failed = e + + if work_queue_size > 100 and work_queue_size >= (redis_work_queue - QUEUE_DROP_AMOUNT): + failed = True + + if failed: + requests.post( + "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME, + auth=("api", settings.MAILGUN_ACCESS_KEY), + data={"from": "NewsBlur Queue Monitor: %s " % (hostname, hostname), + "to": [admin_email], + "subject": "%s work queue rising: %s (from %s)" % (hostname, work_queue_size, redis_work_queue), + "text": "Work queue is rising: %s (from %s)" % (work_queue_size, redis_work_queue)}) + + r_monitor.set(monitor_key, work_queue_size) + r_monitor.expire(monitor_key, 60*60*3) # 3 hours + + print(" ---> Work queue rising! %s %s" % (work_queue_size, failed)) + else: + print(" ---> Work queue OK: %s" % (work_queue_size)) + +if __name__ == '__main__': + main() From e4cfb73b4f912832374cb6353c96b78973450df5 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 20 Jan 2021 13:32:47 -0500 Subject: [PATCH 10/10] Executable monitor work queue. --- fabfile.py | 2 +- utils/monitor_work_queue.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 utils/monitor_work_queue.py diff --git a/fabfile.py b/fabfile.py index a126dbefc..413cc9220 100644 --- a/fabfile.py +++ b/fabfile.py @@ -1493,8 +1493,8 @@ def setup_newsletter_monitor(): @parallel def setup_queue_monitor(): - sudo('/etc/cron.hourly/monitor_work_queue') sudo('ln -fs %s/utils/monitor_work_queue.py /etc/cron.hourly/monitor_work_queue' % env.NEWSBLUR_PATH) + sudo('/etc/cron.hourly/monitor_work_queue') @parallel def setup_redis_monitor(): diff --git a/utils/monitor_work_queue.py b/utils/monitor_work_queue.py old mode 100644 new mode 100755