From b8efcd07135026965817b8c81ffb43d4a1df8543 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 31 Jul 2013 11:17:04 -0700 Subject: [PATCH] Fixing redis/mongo logging accidentally duping logs on fast requests. --- apps/reader/models.py | 8 +++---- apps/reader/views.py | 5 ++-- utils/mongo_raw_log_middleware.py | 40 ++++++++++++++++--------------- utils/redis_raw_log_middleware.py | 16 +++++++------ 4 files changed, 37 insertions(+), 32 deletions(-) diff --git a/apps/reader/models.py b/apps/reader/models.py index 5ea5daaf7..953c8265b 100644 --- a/apps/reader/models.py +++ b/apps/reader/models.py @@ -346,7 +346,7 @@ class UserSubscription(models.Model): return code, message, us @classmethod - def feeds_with_updated_counts(cls, user, feed_ids=None, check_fetch_status=False): + def feeds_with_updated_counts(cls, user, feed_ids=None, check_fetch_status=False, force=False): feeds = {} # Get subscriptions for user @@ -355,15 +355,15 @@ class UserSubscription(models.Model): if feed_ids: user_subs = user_subs.filter(feed__in=feed_ids) - UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD) for i, sub in enumerate(user_subs): # Count unreads if subscription is stale. - if (sub.needs_unread_recalc or + if (force or + sub.needs_unread_recalc or sub.unread_count_updated < UNREAD_CUTOFF or sub.oldest_unread_story_date < UNREAD_CUTOFF): - sub = sub.calculate_feed_scores(silent=True) + sub = sub.calculate_feed_scores(silent=True, force=force) if not sub: continue # TODO: Figure out the correct sub and give it a new feed_id feed_id = sub.feed_id diff --git a/apps/reader/views.py b/apps/reader/views.py index 27a0e2d61..ee181385b 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -424,7 +424,7 @@ def refresh_feeds(request): feeds[feed_id]['favicon_color'] = feed_icons[feed_id].color feeds[feed_id]['favicon_fetching'] = feed.get('favicon_fetching') - user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True) + user_subs = UserSubscription.objects.filter(user=user, active=True) sub_feed_ids = [s.feed_id for s in user_subs] if favicons_fetching: @@ -471,12 +471,13 @@ def interactions_count(request): def feed_unread_count(request): user = request.user feed_ids = request.REQUEST.getlist('feed_id') + force = request.REQUEST.get('force', False) social_feed_ids = [feed_id for feed_id in feed_ids if 'social:' in feed_id] feed_ids = list(set(feed_ids) - set(social_feed_ids)) feeds = {} if feed_ids: - feeds = UserSubscription.feeds_with_updated_counts(user, feed_ids=feed_ids) + feeds = UserSubscription.feeds_with_updated_counts(user, feed_ids=feed_ids, force=force) social_feeds = {} if social_feed_ids: diff --git a/utils/mongo_raw_log_middleware.py b/utils/mongo_raw_log_middleware.py index ea148404a..723785d6e 100644 --- a/utils/mongo_raw_log_middleware.py +++ b/utils/mongo_raw_log_middleware.py @@ -15,16 +15,18 @@ class MongoDumpMiddleware(object): def process_view(self, request, callback, callback_args, callback_kwargs): self._used_msg_ids = [] - if settings.DEBUG: + if settings.DEBUG and not getattr(MongoClient, '_logging', False): # save old methods - self.orig_send_message = \ - MongoClient._send_message - self.orig_send_message_with_response = \ - MongoClient._send_message_with_response - self.orig_rs_send_message = \ - MongoReplicaSetClient._send_message - self.orig_rs_send_message_with_response = \ - MongoReplicaSetClient._send_message_with_response + setattr(MongoClient, '_logging', True) + # # save old methods + # self.orig_send_message = \ + # MongoClient._send_message + # self.orig_send_message_with_response = \ + # MongoClient._send_message_with_response + # self.orig_rs_send_message = \ + # MongoReplicaSetClient._send_message + # self.orig_rs_send_message_with_response = \ + # MongoReplicaSetClient._send_message_with_response # instrument methods to record messages MongoClient._send_message = \ self._instrument(MongoClient._send_message) @@ -37,16 +39,16 @@ class MongoDumpMiddleware(object): return None def process_response(self, request, response): - if settings.DEBUG and hasattr(self, 'orig_send_message') and hasattr(self, 'orig_send_message_with_response'): - # remove instrumentation from pymongo - MongoClient._send_message = \ - self.orig_send_message - MongoClient._send_message_with_response = \ - self.orig_send_message_with_response - MongoReplicaSetClient._send_message = \ - self.orig_rs_send_message - MongoReplicaSetClient._send_message_with_response = \ - self.orig_rs_send_message_with_response + # if settings.DEBUG and hasattr(self, 'orig_send_message') and hasattr(self, 'orig_send_message_with_response'): + # # remove instrumentation from pymongo + # MongoClient._send_message = \ + # self.orig_send_message + # MongoClient._send_message_with_response = \ + # self.orig_send_message_with_response + # MongoReplicaSetClient._send_message = \ + # self.orig_rs_send_message + # MongoReplicaSetClient._send_message_with_response = \ + # self.orig_rs_send_message_with_response return response def _instrument(self, original_method): diff --git a/utils/redis_raw_log_middleware.py b/utils/redis_raw_log_middleware.py index 755df97ea..45fa8b259 100644 --- a/utils/redis_raw_log_middleware.py +++ b/utils/redis_raw_log_middleware.py @@ -10,20 +10,22 @@ class RedisDumpMiddleware(object): raise MiddlewareNotUsed() def process_view(self, request, callback, callback_args, callback_kwargs): - if settings.DEBUG: + if settings.DEBUG and not getattr(Connection, '_logging', False): # save old methods - self.orig_pack_command = \ - Connection.pack_command + setattr(Connection, '_logging', True) + # self.orig_pack_command = \ + # Connection.pack_command # instrument methods to record messages Connection.pack_command = \ self._instrument(Connection.pack_command) return None def process_response(self, request, response): - if settings.DEBUG and hasattr(self, 'orig_pack_command'): - # remove instrumentation from redis - Connection.pack_command = \ - self.orig_pack_command + # if settings.DEBUG and hasattr(self, 'orig_pack_command'): + # # remove instrumentation from redis + # setattr(Connection, '_logging', False) + # Connection.pack_command = \ + # self.orig_pack_command return response def _instrument(self, original_method):