From 2ec10bcf7eb357ee4e49ce245357e19b9944a041 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 29 Oct 2012 12:25:28 -0700 Subject: [PATCH] Readying unread counting to incorporate the new read preference in mongoengine. --- apps/reader/models.py | 28 ++++++++++++++-------------- apps/social/views.py | 4 ++-- fabfile.py | 15 ++++++--------- node/favicons.coffee | 4 +++- node/favicons.js | 6 ++++-- settings.py | 9 +++++++++ utils/feed_fetcher.py | 23 +++++++++++------------ 7 files changed, 49 insertions(+), 40 deletions(-) diff --git a/apps/reader/models.py b/apps/reader/models.py index 69938c4d8..971e83279 100644 --- a/apps/reader/models.py +++ b/apps/reader/models.py @@ -347,7 +347,7 @@ class UserSubscription(models.Model): return data - def calculate_feed_scores(self, silent=False, stories_db=None): + def calculate_feed_scores(self, silent=False, stories=None): # now = datetime.datetime.strptime("2009-07-06 22:30:03", "%Y-%m-%d %H:%M:%S") now = datetime.datetime.now() UNREAD_CUTOFF = now - datetime.timedelta(days=settings.DAYS_OF_UNREAD) @@ -376,23 +376,23 @@ class UserSubscription(models.Model): read_stories = MUserStory.objects(user_id=self.user_id, feed_id=self.feed_id, read_date__gte=self.mark_read_date) - # if not silent: - # logging.info(' ---> [%s] Read stories: %s' % (self.user, datetime.datetime.now() - now)) read_stories_ids = [us.story_id for us in read_stories] - stories_db = stories_db or MStory.objects(story_feed_id=self.feed_id, - story_date__gte=date_delta) - # if not silent: - # logging.info(' ---> [%s] MStory: %s' % (self.user, datetime.datetime.now() - now)) + + if not stories: + stories_db = MStory.objects(story_feed_id=self.feed_id, + story_date__gte=date_delta) + stories = Feed.format_stories(stories_db, self.feed_id) + oldest_unread_story_date = now - unread_stories_db = [] - for story in stories_db: - if story.story_date < date_delta: + unread_stories = [] + for story in stories: + if story['story_date'] < date_delta: continue - if hasattr(story, 'story_guid') and story.story_guid not in read_stories_ids: - unread_stories_db.append(story) + if story.get('story_guid', None) not in read_stories_ids: + unread_stories.append(story) if story.story_date < oldest_unread_story_date: oldest_unread_story_date = story.story_date - stories = Feed.format_stories(unread_stories_db, self.feed_id) + # if not silent: # logging.info(' ---> [%s] Format stories: %s' % (self.user, datetime.datetime.now() - now)) @@ -408,7 +408,7 @@ class UserSubscription(models.Model): 'feed': apply_classifier_feeds(classifier_feeds, self.feed), } - for story in stories: + for story in unread_stories: scores.update({ 'author' : apply_classifier_authors(classifier_authors, story), 'tags' : apply_classifier_tags(classifier_tags, story), diff --git a/apps/social/views.py b/apps/social/views.py index ecfd0376f..cd8b7f026 100644 --- a/apps/social/views.py +++ b/apps/social/views.py @@ -8,7 +8,7 @@ from django.core.urlresolvers import reverse from django.contrib.auth.models import User from django.contrib.sites.models import Site from django.template.loader import render_to_string -from django.http import HttpResponse, HttpResponseRedirect, Http404, Http403 +from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseForbidden from django.conf import settings from django.template import RequestContext from django.utils import feedgenerator @@ -1110,7 +1110,7 @@ def shared_stories_rss_feed(request, user_id, username): current_site = current_site and current_site.domain if social_profile.private: - return Http403 + return HttpResponseForbidden data = {} data['title'] = social_profile.title diff --git a/fabfile.py b/fabfile.py index be283ae78..5b51d6502 100644 --- a/fabfile.py +++ b/fabfile.py @@ -455,12 +455,11 @@ def setup_mongoengine(): with cd(env.VENDOR_PATH): with settings(warn_only=True): run('rm -fr mongoengine') - run('git clone https://github.com/mongoengine/mongoengine.git') + run('git clone https://github.com/MongoEngine/mongoengine.git') sudo('rm -f /usr/local/lib/python2.7/dist-packages/mongoengine') + sudo('rm -f /usr/local/lib/python2.7/dist-packages/mongoengine-*') sudo('ln -s %s /usr/local/lib/python2.7/dist-packages/mongoengine' % os.path.join(env.VENDOR_PATH, 'mongoengine/mongoengine')) - with cd(os.path.join(env.VENDOR_PATH, 'mongoengine')): - run('git checkout -b dev origin/dev') def setup_pymongo_repo(): with cd(env.VENDOR_PATH): @@ -472,12 +471,10 @@ def setup_pymongo_repo(): def setup_forked_mongoengine(): with cd(os.path.join(env.VENDOR_PATH, 'mongoengine')): with settings(warn_only=True): - run('git checkout master') - run('git branch -D dev') - run('git remote add %s git://github.com/samuelclay/mongoengine.git' % env.user) - run('git fetch %s' % env.user) - run('git checkout -b dev %s/dev' % env.user) - run('git pull %s dev' % env.user) + run('git remote add clay https://github.com/samuelclay/mongoengine.git') + run('git pull') + run('git fetch clay') + run('git checkout -b clay_master clay/master') def switch_forked_mongoengine(): with cd(os.path.join(env.VENDOR_PATH, 'mongoengine')): diff --git a/node/favicons.coffee b/node/favicons.coffee index 688e1e341..f86d5c0b0 100644 --- a/node/favicons.coffee +++ b/node/favicons.coffee @@ -2,7 +2,9 @@ express = require 'express' mongo = require 'mongodb' MONGODB_SERVER = if process.env.NODE_ENV == 'development' then 'localhost' else 'db04' -server = new mongo.Server(MONGODB_SERVER, 27017, +MONGODB_PORT = parseInt(process.env.MONGODB_PORT or 27017, 10) + +server = new mongo.Server(MONGODB_SERVER, MONGODB_PORT, auto_reconnect: true poolSize: 12) db = new mongo.Db('newsblur', server) diff --git a/node/favicons.js b/node/favicons.js index 1c548b0b8..f73b35bff 100644 --- a/node/favicons.js +++ b/node/favicons.js @@ -1,6 +1,6 @@ // Generated by CoffeeScript 1.4.0 (function() { - var MONGODB_SERVER, app, db, express, mongo, server, + var MONGODB_PORT, MONGODB_SERVER, app, db, express, mongo, server, _this = this; express = require('express'); @@ -9,7 +9,9 @@ MONGODB_SERVER = process.env.NODE_ENV === 'development' ? 'localhost' : 'db04'; - server = new mongo.Server(MONGODB_SERVER, 27017, { + MONGODB_PORT = parseInt(process.env.MONGODB_PORT || 27017, 10); + + server = new mongo.Server(MONGODB_SERVER, MONGODB_PORT, { auto_reconnect: true, poolSize: 12 }); diff --git a/settings.py b/settings.py index a6b44045f..2acf85a64 100644 --- a/settings.py +++ b/settings.py @@ -469,8 +469,16 @@ MONGO_DB_DEFAULTS = { 'alias': 'default', } MONGO_DB = dict(MONGO_DB_DEFAULTS, **MONGO_DB) + +# if MONGO_DB.get('read_preference', pymongo.ReadPreference.PRIMARY) != pymongo.ReadPreference.PRIMARY: +# MONGO_PRIMARY_DB = MONGO_DB.copy() +# MONGO_PRIMARY_DB.update(read_preference=pymongo.ReadPreference.PRIMARY) +# MONGOPRIMARYDB = connect(MONGO_PRIMARY_DB.pop('name'), **MONGO_PRIMARY_DB) +# else: +# MONGOPRIMARYDB = MONGODB MONGODB = connect(MONGO_DB.pop('name'), **MONGO_DB) + MONGO_ANALYTICS_DB_DEFAULTS = { 'name': 'nbanalytics', 'host': 'db02:27017', @@ -479,6 +487,7 @@ MONGO_ANALYTICS_DB_DEFAULTS = { MONGO_ANALYTICS_DB = dict(MONGO_ANALYTICS_DB_DEFAULTS, **MONGO_ANALYTICS_DB) MONGOANALYTICSDB = connect(MONGO_ANALYTICS_DB.pop('name'), **MONGO_ANALYTICS_DB) + # ========= # = Redis = # ========= diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index 5918fbe71..63751b66b 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -229,11 +229,6 @@ class ProcessFeed: story_feed_id=self.feed_id ).limit(max(int(len(story_guids)*1.5), 10))) - # MStory.objects( - # (Q(story_date__gte=start_date) & Q(story_date__lte=end_date)) - # | (Q(story_guid__in=story_guids)), - # story_feed=self.feed - # ).order_by('-story_date') ret_values = self.feed.add_update_stories(stories, existing_stories, verbose=self.options['verbose']) @@ -513,28 +508,32 @@ class Dispatcher: active=True, user__profile__last_seen_on__gte=UNREAD_CUTOFF)\ .order_by('-last_read_date') - + + if not user_subs.count(): + return + for sub in user_subs: if not sub.needs_unread_recalc: sub.needs_unread_recalc = True sub.save() if self.options['compute_scores']: - stories_db = MStory.objects(story_feed_id=feed.pk, - story_date__gte=UNREAD_CUTOFF) + stories = MStory.objects(story_feed_id=feed.pk, + story_date__gte=UNREAD_CUTOFF) + stories = Feed.format_stories(stories, feed.pk) logging.debug(u' ---> [%-30s] ~FYComputing scores: ~SB%s stories~SN with ~SB%s subscribers ~SN(%s/%s/%s)' % ( - feed.title[:30], stories_db.count(), user_subs.count(), + feed.title[:30], len(stories), user_subs.count(), feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers)) - self.calculate_feed_scores_with_stories(user_subs, stories_db) + self.calculate_feed_scores_with_stories(user_subs, stories) elif self.options.get('mongodb_replication_lag'): logging.debug(u' ---> [%-30s] ~BR~FYSkipping computing scores: ~SB%s seconds~SN of mongodb lag' % ( feed.title[:30], self.options.get('mongodb_replication_lag'))) @timelimit(10) - def calculate_feed_scores_with_stories(self, user_subs, stories_db): + def calculate_feed_scores_with_stories(self, user_subs, stories): for sub in user_subs: silent = False if self.options['verbose'] >= 2 else True - sub.calculate_feed_scores(silent=silent, stories_db=stories_db) + sub.calculate_feed_scores(silent=silent, stories=stories) def add_jobs(self, feeds_queue, feeds_count=1): """ adds a feed processing job to the pool