From b3944cab070c145a3092b438b779cffadee3d097 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 12 Jan 2021 14:50:27 -0500 Subject: [PATCH 01/19] Don't need full health checks in debug. --- config/debug_haproxy.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/debug_haproxy.conf b/config/debug_haproxy.conf index 956191f6e..38c7cd9d2 100644 --- a/config/debug_haproxy.conf +++ b/config/debug_haproxy.conf @@ -74,8 +74,8 @@ backend imageproxy server imageproxy01 imageproxy:80 check inter 2000ms backend push - option httpchk GET /_haproxychk - http-check expect rstatus 200|503 + # option httpchk GET /_haproxychk + # http-check expect rstatus 200|503 server push 127.0.0.1:8000 check inter 2000ms backend node_socket From bbd6b9d4743d5b2be42b680b9bd94cb32b319179 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 6 Jan 2021 18:07:52 -0500 Subject: [PATCH 02/19] Revert "Removing unused types key from index." This reverts commit 6b384b51a2086e0e91ec30e3a1d5db9c125d8656. --- apps/notifications/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/notifications/models.py b/apps/notifications/models.py index 91ae0081f..ac0690ec0 100644 --- a/apps/notifications/models.py +++ b/apps/notifications/models.py @@ -38,6 +38,7 @@ class MUserNotificationTokens(mongo.Document): 'collection': 'notification_tokens', 'indexes': [{'fields': ['user_id'], 'unique': True, + 'types': False, }], 'allow_inheritance': False, } From a4f1443c5c73190971c135f2765be0c90f77c619 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:43:18 -0500 Subject: [PATCH 03/19] Only warn about falling feed fetches when actually falling. --- utils/monitor_task_fetches.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index 84c565b09..561d0317a 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -18,17 +18,24 @@ def main(): admin_email = settings.ADMINS[0][1] failed = False feeds_fetched = 0 - + FETCHES_DROP_AMOUNT = 0 + monitor_key = "Monitor:task_fetches" + r = redis.Redis(connection_pool=settings.REDIS_ANALYTICS_POOL) + try: client = pymongo.MongoClient('mongodb://%s' % settings.MONGO_DB['host']) feeds_fetched = client.newsblur.statistics.find_one({"key": "feeds_fetched"})['value'] + redis_task_fetches = int(r.get(monitor_key, feeds_fetched)) except Exception, e: failed = e - if feeds_fetched < 5000000: + if feeds_fetched < 5000000 and feeds_fetched <= (redis_task_fetches - FETCHES_DROP_AMOUNT): failed = True - + if failed: + r.set(monitor_key, feeds_fetched) + r.expire(monitor_key, 60*60*3) # 3 hours + requests.post( "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME, auth=("api", settings.MAILGUN_ACCESS_KEY), From c9c891e44a2378b481742d40766297cb19d3c8c3 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:44:19 -0500 Subject: [PATCH 04/19] Needs redis --- utils/monitor_task_fetches.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index 561d0317a..b58abe744 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -7,6 +7,7 @@ import subprocess import requests from newsblur import settings import socket +import redis import pymongo def main(): @@ -35,7 +36,7 @@ def main(): if failed: r.set(monitor_key, feeds_fetched) r.expire(monitor_key, 60*60*3) # 3 hours - + requests.post( "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME, auth=("api", settings.MAILGUN_ACCESS_KEY), From 238aa3916acbc6c3c06e0c7f73bb018c3793f3af Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:45:35 -0500 Subject: [PATCH 05/19] Handling missing values --- utils/monitor_task_fetches.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index b58abe744..b1a3b455f 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -20,6 +20,7 @@ def main(): failed = False feeds_fetched = 0 FETCHES_DROP_AMOUNT = 0 + redis_task_fetches = 0 monitor_key = "Monitor:task_fetches" r = redis.Redis(connection_pool=settings.REDIS_ANALYTICS_POOL) @@ -34,9 +35,6 @@ def main(): failed = True if failed: - r.set(monitor_key, feeds_fetched) - r.expire(monitor_key, 60*60*3) # 3 hours - requests.post( "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME, auth=("api", settings.MAILGUN_ACCESS_KEY), @@ -44,6 +42,10 @@ def main(): "to": [admin_email], "subject": "%s feeds fetched falling: %s" % (hostname, feeds_fetched), "text": "Feed fetches are falling: %s" % (feeds_fetched)}) + + r.set(monitor_key, feeds_fetched) + r.expire(monitor_key, 60*60*3) # 3 hours + print(" ---> Feeds fetched falling! %s" % (feeds_fetched)) else: print(" ---> Feeds fetched OK: %s" % (feeds_fetched)) From b58ab9f9cd2543ccee3104ffb65ed7222a307ea4 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:46:52 -0500 Subject: [PATCH 06/19] Spitting out error on failure. --- utils/monitor_task_fetches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index b1a3b455f..aeed39e69 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -46,7 +46,7 @@ def main(): r.set(monitor_key, feeds_fetched) r.expire(monitor_key, 60*60*3) # 3 hours - print(" ---> Feeds fetched falling! %s" % (feeds_fetched)) + print(" ---> Feeds fetched falling! %s %s" % (feeds_fetched, failed)) else: print(" ---> Feeds fetched OK: %s" % (feeds_fetched)) From d747acc5d0a0657bff280d1a75a65e20dcde91ca Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:47:17 -0500 Subject: [PATCH 07/19] No default value --- utils/monitor_task_fetches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index aeed39e69..8cc3624e0 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -27,7 +27,7 @@ def main(): try: client = pymongo.MongoClient('mongodb://%s' % settings.MONGO_DB['host']) feeds_fetched = client.newsblur.statistics.find_one({"key": "feeds_fetched"})['value'] - redis_task_fetches = int(r.get(monitor_key, feeds_fetched)) + redis_task_fetches = int(r.get(monitor_key)) except Exception, e: failed = e From 047648b508aa6f4605cb162730bcb4aaaaa064fe Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 17:49:12 -0500 Subject: [PATCH 08/19] More descriptive failure email --- utils/monitor_task_fetches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index 8cc3624e0..a16e2dec4 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -40,8 +40,8 @@ def main(): auth=("api", settings.MAILGUN_ACCESS_KEY), data={"from": "NewsBlur Task Monitor: %s " % (hostname, hostname), "to": [admin_email], - "subject": "%s feeds fetched falling: %s" % (hostname, feeds_fetched), - "text": "Feed fetches are falling: %s" % (feeds_fetched)}) + "subject": "%s feeds fetched falling: %s (from %s)" % (hostname, feeds_fetched, redis_task_fetches), + "text": "Feed fetches are falling: %s (from %s)" % (feeds_fetched, redis_task_fetches)}) r.set(monitor_key, feeds_fetched) r.expire(monitor_key, 60*60*3) # 3 hours From 7d218a652ad63f6ab4c930cd16db14f6774785ba Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 18 Jan 2021 20:16:25 -0500 Subject: [PATCH 09/19] Pointing to the new ssl cert. --- config/haproxy.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/haproxy.conf.template b/config/haproxy.conf.template index 18597d073..a542be20f 100644 --- a/config/haproxy.conf.template +++ b/config/haproxy.conf.template @@ -166,7 +166,7 @@ backend maintenance {{ maintenance }} listen stats - bind :1936 ssl crt newsblur.pem + bind :1936 ssl crt newsblur.com.crt stats enable stats hide-version stats realm Haproxy\ Statistics From b54c98198ea29986f53087051203f185a23f3b34 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 20 Jan 2021 13:29:12 -0500 Subject: [PATCH 10/19] Adding monitor for work queue. --- fabfile.py | 12 ++++++--- utils/monitor_work_queue.py | 54 +++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 utils/monitor_work_queue.py diff --git a/fabfile.py b/fabfile.py index 064f94252..a126dbefc 100644 --- a/fabfile.py +++ b/fabfile.py @@ -828,15 +828,14 @@ def copy_certificates(): run('ln -fs %s %s' % (privkey_path, os.path.join(cert_path, 'newsblur.com.crt.key'))) # HAProxy put(os.path.join(env.SECRETS_PATH, 'certificates/comodo/dhparams.pem'), cert_path) put(os.path.join(env.SECRETS_PATH, 'certificates/ios/aps_development.pem'), cert_path) + + # Export aps.cer from Apple issued certificate using Keychain Assistant # openssl x509 -in aps.cer -inform DER -outform PEM -out aps.pem put(os.path.join(env.SECRETS_PATH, 'certificates/ios/aps.pem'), cert_path) # Export aps.p12 from aps.cer using Keychain Assistant # openssl pkcs12 -in aps.p12 -out aps.p12.pem -nodes put(os.path.join(env.SECRETS_PATH, 'certificates/ios/aps.p12.pem'), cert_path) - # run('cat %s/newsblur.com.crt > %s/newsblur.pem' % (cert_path, cert_path)) - # run('echo "\n" >> %s/newsblur.pem' % (cert_path)) - # run('cat %s/newsblur.com.key >> %s/newsblur.pem' % (cert_path, cert_path)) - + def setup_certbot(): sudo('snap install --classic certbot') sudo('snap set certbot trust-plugin-with-root=ok') @@ -1492,6 +1491,11 @@ def setup_newsletter_monitor(): sudo('ln -fs %s/utils/monitor_newsletter_delivery.py /etc/cron.hourly/monitor_newsletter_delivery' % env.NEWSBLUR_PATH) sudo('/etc/cron.hourly/monitor_newsletter_delivery') +@parallel +def setup_queue_monitor(): + sudo('/etc/cron.hourly/monitor_work_queue') + sudo('ln -fs %s/utils/monitor_work_queue.py /etc/cron.hourly/monitor_work_queue' % env.NEWSBLUR_PATH) + @parallel def setup_redis_monitor(): run('sleep 5') # Wait for redis to startup so the log file is there diff --git a/utils/monitor_work_queue.py b/utils/monitor_work_queue.py new file mode 100644 index 000000000..0c5ba2deb --- /dev/null +++ b/utils/monitor_work_queue.py @@ -0,0 +1,54 @@ +#!/srv/newsblur/venv/newsblur/bin/python + +import sys +sys.path.append('/srv/newsblur') + +import subprocess +import requests +from newsblur import settings +import socket +import redis +import pymongo + +def main(): + df = subprocess.Popen(["df", "/"], stdout=subprocess.PIPE) + output = df.communicate()[0] + device, size, used, available, percent, mountpoint = output.split("\n")[1].split() + hostname = socket.gethostname() + percent = int(percent.strip('%')) + admin_email = settings.ADMINS[0][1] + failed = False + work_queue_size = 0 + QUEUE_DROP_AMOUNT = 0 + redis_work_queue = 0 + monitor_key = "Monitor:work_queue" + r_monitor = redis.Redis(connection_pool=settings.REDIS_ANALYTICS_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) + + try: + work_queue_size = int(r.llen("work_queue")) + redis_work_queue = int(r_monitor.get(monitor_key)) + except Exception, e: + failed = e + + if work_queue_size > 100 and work_queue_size >= (redis_work_queue - QUEUE_DROP_AMOUNT): + failed = True + + if failed: + requests.post( + "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME, + auth=("api", settings.MAILGUN_ACCESS_KEY), + data={"from": "NewsBlur Queue Monitor: %s " % (hostname, hostname), + "to": [admin_email], + "subject": "%s work queue rising: %s (from %s)" % (hostname, work_queue_size, redis_work_queue), + "text": "Work queue is rising: %s (from %s)" % (work_queue_size, redis_work_queue)}) + + r_monitor.set(monitor_key, work_queue_size) + r_monitor.expire(monitor_key, 60*60*3) # 3 hours + + print(" ---> Work queue rising! %s %s" % (work_queue_size, failed)) + else: + print(" ---> Work queue OK: %s" % (work_queue_size)) + +if __name__ == '__main__': + main() From e4cfb73b4f912832374cb6353c96b78973450df5 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 20 Jan 2021 13:32:47 -0500 Subject: [PATCH 11/19] Executable monitor work queue. --- fabfile.py | 2 +- utils/monitor_work_queue.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 utils/monitor_work_queue.py diff --git a/fabfile.py b/fabfile.py index a126dbefc..413cc9220 100644 --- a/fabfile.py +++ b/fabfile.py @@ -1493,8 +1493,8 @@ def setup_newsletter_monitor(): @parallel def setup_queue_monitor(): - sudo('/etc/cron.hourly/monitor_work_queue') sudo('ln -fs %s/utils/monitor_work_queue.py /etc/cron.hourly/monitor_work_queue' % env.NEWSBLUR_PATH) + sudo('/etc/cron.hourly/monitor_work_queue') @parallel def setup_redis_monitor(): diff --git a/utils/monitor_work_queue.py b/utils/monitor_work_queue.py old mode 100644 new mode 100755 From 3811d288d8eae9839df619519ed9894c87f63b09 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 20 Jan 2021 17:20:01 -0500 Subject: [PATCH 12/19] Limiting work queue checks. --- utils/monitor_work_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/monitor_work_queue.py b/utils/monitor_work_queue.py index 0c5ba2deb..2b80aed05 100755 --- a/utils/monitor_work_queue.py +++ b/utils/monitor_work_queue.py @@ -31,7 +31,7 @@ def main(): except Exception, e: failed = e - if work_queue_size > 100 and work_queue_size >= (redis_work_queue - QUEUE_DROP_AMOUNT): + if work_queue_size > 100 and work_queue_size > (redis_work_queue + QUEUE_DROP_AMOUNT): failed = True if failed: From e15c5a358f1c238ff19f3ab97182e1dceb0c5723 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 20 Jan 2021 21:25:01 -0500 Subject: [PATCH 13/19] Handling work queue missing error. --- utils/monitor_task_fetches.py | 2 +- utils/monitor_work_queue.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index a16e2dec4..6aca3760f 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -41,7 +41,7 @@ def main(): data={"from": "NewsBlur Task Monitor: %s " % (hostname, hostname), "to": [admin_email], "subject": "%s feeds fetched falling: %s (from %s)" % (hostname, feeds_fetched, redis_task_fetches), - "text": "Feed fetches are falling: %s (from %s)" % (feeds_fetched, redis_task_fetches)}) + "text": "Feed fetches are falling: %s (from %s) %s" % (feeds_fetched, redis_task_fetches, failed)}) r.set(monitor_key, feeds_fetched) r.expire(monitor_key, 60*60*3) # 3 hours diff --git a/utils/monitor_work_queue.py b/utils/monitor_work_queue.py index 2b80aed05..0f9ebe908 100755 --- a/utils/monitor_work_queue.py +++ b/utils/monitor_work_queue.py @@ -27,7 +27,7 @@ def main(): try: work_queue_size = int(r.llen("work_queue")) - redis_work_queue = int(r_monitor.get(monitor_key)) + redis_work_queue = int(r_monitor.get(monitor_key) or 0) except Exception, e: failed = e @@ -41,7 +41,7 @@ def main(): data={"from": "NewsBlur Queue Monitor: %s " % (hostname, hostname), "to": [admin_email], "subject": "%s work queue rising: %s (from %s)" % (hostname, work_queue_size, redis_work_queue), - "text": "Work queue is rising: %s (from %s)" % (work_queue_size, redis_work_queue)}) + "text": "Work queue is rising: %s (from %s) %s" % (work_queue_size, redis_work_queue, failed)}) r_monitor.set(monitor_key, work_queue_size) r_monitor.expire(monitor_key, 60*60*3) # 3 hours From 6f9dbded8eb134d6901873b4aa16d3043e5fc6c4 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 20 Jan 2021 21:28:01 -0500 Subject: [PATCH 14/19] Attempting to see iOS premium status over email. --- apps/profile/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/profile/views.py b/apps/profile/views.py index 29cb930df..d2dcf5402 100644 --- a/apps/profile/views.py +++ b/apps/profile/views.py @@ -710,8 +710,8 @@ def email_optout(request): @json.json_view def ios_subscription_status(request): logging.debug(" ---> iOS Subscription Status: %s" % request.body) - - subject = "iOS Subscription Status" + data = json.decode(request.body) + subject = "iOS Subscription Status: %s" % data.get('notification_type', "[missing]") message = """%s""" % (request.body) mail_admins(subject, message) From d2f1a417c9e156cf5bf591cbf387778132bddc95 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 22 Jan 2021 19:31:49 -0500 Subject: [PATCH 15/19] Fix task fetch monitor. --- utils/monitor_task_fetches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index 6aca3760f..f934c585e 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -27,7 +27,7 @@ def main(): try: client = pymongo.MongoClient('mongodb://%s' % settings.MONGO_DB['host']) feeds_fetched = client.newsblur.statistics.find_one({"key": "feeds_fetched"})['value'] - redis_task_fetches = int(r.get(monitor_key)) + redis_task_fetches = int(r.get(monitor_key) or 0) except Exception, e: failed = e From 33183967b3688cfe4537862eed49d7e4370c1b13 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sat, 23 Jan 2021 14:20:19 -0500 Subject: [PATCH 16/19] Checking if a feed has any notifications on it, set a max of 30 min fetch (yes this is a pretty good way of getting your feeds to fetch faster, but youll have to deal with the notifications). --- apps/rss_feeds/models.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 9707dabf1..b70baeef6 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -2099,12 +2099,15 @@ class Feed(models.Model): if self.min_to_decay and not force and not premium_speed: return self.min_to_decay + from apps.notifications.models import MUserFeedNotification + if premium_speed: self.active_premium_subscribers += 1 spd = self.stories_last_month / 30.0 subs = (self.active_premium_subscribers + ((self.active_subscribers - self.active_premium_subscribers) / 10.0)) + notification_count = MUserFeedNotification.objects.filter(feed_id=self.pk).count() # Calculate sub counts: # SELECT COUNT(*) FROM feeds WHERE active_premium_subscribers > 10 AND stories_last_month >= 30; # SELECT COUNT(*) FROM feeds WHERE active_premium_subscribers > 1 AND active_premium_subscribers < 10 AND stories_last_month >= 30; @@ -2163,6 +2166,10 @@ class Feed(models.Model): if len(fetch_history['push_history']): total = total * 12 + # Any notifications means a 30 min minumum + if notification_count > 0: + total = min(total, 30) + # 4 hour max for premiums, 48 hour max for free if subs >= 1: total = min(total, 60*4*1) From 6badcaf2943e2ce6f1d9bb77e2895953b0d08e26 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sat, 23 Jan 2021 16:48:13 -0500 Subject: [PATCH 17/19] Removing old types field. Needed to upgrade to mongo 3.4. --- apps/notifications/models.py | 2 -- apps/profile/models.py | 1 - apps/rss_feeds/models.py | 2 -- 3 files changed, 5 deletions(-) diff --git a/apps/notifications/models.py b/apps/notifications/models.py index ac0690ec0..4c7e1e2e7 100644 --- a/apps/notifications/models.py +++ b/apps/notifications/models.py @@ -38,7 +38,6 @@ class MUserNotificationTokens(mongo.Document): 'collection': 'notification_tokens', 'indexes': [{'fields': ['user_id'], 'unique': True, - 'types': False, }], 'allow_inheritance': False, } @@ -71,7 +70,6 @@ class MUserFeedNotification(mongo.Document): 'indexes': ['feed_id', {'fields': ['user_id', 'feed_id'], 'unique': True, - 'types': False, }], 'allow_inheritance': False, } diff --git a/apps/profile/models.py b/apps/profile/models.py index 4ffa62466..259f4a70c 100644 --- a/apps/profile/models.py +++ b/apps/profile/models.py @@ -1219,7 +1219,6 @@ class MEmailUnsubscribe(mongo.Document): 'indexes': ['user_id', {'fields': ['user_id', 'email_type'], 'unique': True, - 'types': False, }], } diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index b70baeef6..2cd4af372 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -2426,7 +2426,6 @@ class MStory(mongo.Document): 'indexes': [('story_feed_id', '-story_date'), {'fields': ['story_hash'], 'unique': True, - 'types': False, }], 'ordering': ['-story_date'], 'allow_inheritance': False, @@ -3184,7 +3183,6 @@ class MSavedSearch(mongo.Document): 'indexes': ['user_id', {'fields': ['user_id', 'feed_id', 'query'], 'unique': True, - 'types': False, }], 'ordering': ['query'], 'allow_inheritance': False, From be26dbcd22aacc60a2d1cf3e685635f81474c65d Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sat, 23 Jan 2021 18:01:19 -0500 Subject: [PATCH 18/19] Use email if username doesnt work. --- apps/profile/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/profile/models.py b/apps/profile/models.py index 259f4a70c..6ace8692b 100644 --- a/apps/profile/models.py +++ b/apps/profile/models.py @@ -488,6 +488,8 @@ class Profile(models.Model): @property def latest_paypal_email(self): ipn = PayPalIPN.objects.filter(custom=self.user.username) + if not len(ipn): + ipn = PayPalIPN.objects.filter(payer_email=self.user.email) if not len(ipn): return From bfc75d6232b2a6476bc9bf8868471042f7d98200 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sat, 23 Jan 2021 19:27:54 -0500 Subject: [PATCH 19/19] Upgrading to mongo 3.4. --- fabfile.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fabfile.py b/fabfile.py index 413cc9220..6659452f5 100644 --- a/fabfile.py +++ b/fabfile.py @@ -492,8 +492,8 @@ def virtualenv(): def setup_pip(): with cd(env.VENDOR_PATH), settings(warn_only=True): - run('curl https://bootstrap.pypa.io/get-pip.py --output get-pip.py') - sudo('python2 get-pip.py') + run('curl https://bootstrap.pypa.io/2.6/get-pip.py | sudo python2') + # sudo('python2 get-pip.py') @parallel @@ -509,9 +509,9 @@ def pip(): sudo('mkswap /swapfile') sudo('swapon /swapfile') sudo('chown %s.%s -R %s' % (env.user, env.user, os.path.join(env.NEWSBLUR_PATH, 'venv'))) - run('easy_install -U pip') - run('pip install --upgrade pip') - run('pip install --upgrade setuptools') + # run('easy_install -U pip') + # run('pip install --upgrade pip') + # run('pip install --upgrade setuptools') run('pip install -r requirements.txt') if role == "task": with settings(warn_only=True): @@ -1214,7 +1214,7 @@ def disable_thp(): sudo('update-rc.d disable-transparent-hugepages defaults') def setup_mongo(): - MONGODB_VERSION = "3.2.22" + MONGODB_VERSION = "3.4.24" pull() disable_thp() sudo('systemctl enable rc-local.service') # Enable rc.local @@ -1225,11 +1225,11 @@ def setup_mongo(): echo never > /sys/kernel/mm/transparent_hugepage/defrag\n\ fi\n\n\ exit 0" | sudo tee /etc/rc.local') - sudo('curl -fsSL https://www.mongodb.org/static/pgp/server-3.2.asc | sudo apt-key add -') + sudo('curl -fsSL https://www.mongodb.org/static/pgp/server-3.4.asc | sudo apt-key add -') # sudo('echo "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen" | sudo tee /etc/apt/sources.list.d/mongodb.list') # sudo('echo "\ndeb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen" | sudo tee -a /etc/apt/sources.list') # sudo('echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list') - sudo('echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list') + sudo('echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.4.list') sudo('apt-get update') sudo('apt-get install -y mongodb-org=%s mongodb-org-server=%s mongodb-org-shell=%s mongodb-org-mongos=%s mongodb-org-tools=%s' % (MONGODB_VERSION, MONGODB_VERSION, MONGODB_VERSION, MONGODB_VERSION, MONGODB_VERSION))