mirror of
https://github.com/samuelclay/NewsBlur.git
synced 2025-09-18 21:50:56 +00:00
Migration to make loading read stories faster, by using a bounding box on read stories, far fewer can be fetched.
This commit is contained in:
Samuel Clay
parent
442012fdeb
commit
af9f9beda5
3 changed files with 134 additions and 14 deletions
121
apps/reader/migrations/0010_story_dates_on_userstories.py
Normal file
121
apps/reader/migrations/0010_story_dates_on_userstories.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
# encoding: utf-8
|
||||
import datetime
|
||||
from south.db import db
|
||||
from south.v2 import DataMigration
|
||||
from django.db import models
|
||||
from apps.reader.models import MUserStory
|
||||
|
||||
class Migration(DataMigration):
|
||||
|
||||
def forwards(self, orm):
|
||||
"Write your forwards methods here."
|
||||
userstories = MUserStory.objects.all()
|
||||
print "%s userstories" % userstories.count()
|
||||
|
||||
for i, userstory in enumerate(userstories):
|
||||
print '%s: %s' % (i, userstory.story.story_date)
|
||||
userstory.story_date = userstory.story.story_date
|
||||
userstory.save()
|
||||
|
||||
def backwards(self, orm):
|
||||
"Write your backwards methods here."
|
||||
|
||||
|
||||
models = {
|
||||
'auth.group': {
|
||||
'Meta': {'object_name': 'Group'},
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
|
||||
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
|
||||
},
|
||||
'auth.permission': {
|
||||
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
|
||||
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
|
||||
},
|
||||
'auth.user': {
|
||||
'Meta': {'object_name': 'User'},
|
||||
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
|
||||
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
|
||||
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
|
||||
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
|
||||
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
|
||||
},
|
||||
'contenttypes.contenttype': {
|
||||
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
|
||||
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
|
||||
},
|
||||
'reader.feature': {
|
||||
'Meta': {'ordering': "['-date']", 'object_name': 'Feature'},
|
||||
'date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
|
||||
},
|
||||
'reader.usersubscription': {
|
||||
'Meta': {'unique_together': "(('user', 'feed'),)", 'object_name': 'UserSubscription'},
|
||||
'active': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'subscribers'", 'to': "orm['rss_feeds.Feed']"}),
|
||||
'feed_opens': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'is_trained': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'last_read_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2011, 8, 4, 20, 5, 48, 367591)'}),
|
||||
'mark_read_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2011, 8, 4, 20, 5, 48, 367591)'}),
|
||||
'needs_unread_recalc': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'oldest_unread_story_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'unread_count_negative': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'unread_count_neutral': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'unread_count_positive': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'unread_count_updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'subscriptions'", 'to': "orm['auth.User']"}),
|
||||
'user_title': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'})
|
||||
},
|
||||
'reader.usersubscriptionfolders': {
|
||||
'Meta': {'object_name': 'UserSubscriptionFolders'},
|
||||
'folders': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'unique': 'True'})
|
||||
},
|
||||
'rss_feeds.feed': {
|
||||
'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
|
||||
'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
|
||||
'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
|
||||
'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
|
||||
'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
|
||||
'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
|
||||
'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
|
||||
'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'feed_address': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '255'}),
|
||||
'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
|
||||
'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
|
||||
'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
|
||||
'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
|
||||
'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
|
||||
'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
|
||||
'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
|
||||
'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
|
||||
'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
|
||||
}
|
||||
}
|
||||
|
||||
complete_apps = ['reader']
|
||||
|
|
@ -257,11 +257,12 @@ class MUserStory(mongo.Document):
|
|||
user_id = mongo.IntField()
|
||||
feed_id = mongo.IntField()
|
||||
read_date = mongo.DateTimeField()
|
||||
story_date = mongo.DateTimeField()
|
||||
story = mongo.ReferenceField(MStory, unique_with=('user_id', 'feed_id'))
|
||||
|
||||
meta = {
|
||||
'collection': 'userstories',
|
||||
'indexes': [('user_id', 'feed_id'), ('feed_id', 'read_date')],
|
||||
'indexes': [('user_id', 'feed_id'), ('feed_id', 'read_date'), ('feed_id', 'story_date')],
|
||||
'allow_inheritance': False,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -313,10 +313,11 @@ def load_single_feed(request, feed_id):
|
|||
|
||||
usersub = UserSubscription.objects.get(user=user, feed=feed)
|
||||
userstories = []
|
||||
if usersub:
|
||||
if usersub and stories:
|
||||
userstories_db = MUserStory.objects(user_id=user.pk,
|
||||
feed_id=feed.pk,
|
||||
read_date__gte=usersub.mark_read_date).only('story')
|
||||
story_date__lte=stories[0]['story_date'],
|
||||
story_date__gte=stories[-1]['story_date']).only('story')
|
||||
starred_stories = MStarredStory.objects(user_id=user.pk, story_feed_id=feed_id).only('story_guid', 'starred_date')
|
||||
starred_stories = dict([(story.story_guid, story.starred_date) for story in starred_stories])
|
||||
|
||||
|
|
@ -364,18 +365,15 @@ def load_single_feed(request, feed_id):
|
|||
if usersub:
|
||||
usersub.feed_opens += 1
|
||||
usersub.save()
|
||||
timediff = time.time()-start
|
||||
last_update = relative_timesince(feed.last_update)
|
||||
logging.user(request.user, "~FYLoading feed: ~SB%s%s ~SN(%.4s seconds)" % (
|
||||
feed, ('~SN/p%s' % page) if page > 1 else '', timediff))
|
||||
FeedLoadtime.objects.create(feed=feed, loadtime=timediff)
|
||||
|
||||
if timediff >= 1:
|
||||
diff1 = checkpoint1-start
|
||||
diff2 = checkpoint2-start
|
||||
diff3 = checkpoint3-start
|
||||
logging.user(request.user, "~FYSlow feed load: ~SB%.4s/%.4s(%s)/%.4s" % (
|
||||
diff1, diff2, userstories_db and userstories_db.count(), diff3))
|
||||
timediff = time.time()-start
|
||||
last_update = relative_timesince(feed.last_update)
|
||||
logging.user(request.user, "~FYLoading feed: ~SB%s%s ~SN(%.4s seconds, ~SB%.4s/%.4s(%s)/%.4s~SN)" % (
|
||||
feed.feed_title[:32], ('~SN/p%s' % page) if page > 1 else '', timediff,
|
||||
diff1, diff2, userstories_db and userstories_db.count() or '~SN0~SB', diff3))
|
||||
FeedLoadtime.objects.create(feed=feed, loadtime=timediff)
|
||||
|
||||
data = dict(stories=stories,
|
||||
feed_tags=feed_tags,
|
||||
|
|
@ -622,13 +620,14 @@ def mark_story_as_read(request):
|
|||
continue
|
||||
now = datetime.datetime.utcnow()
|
||||
date = now if now > story.story_date else story.story_date # For handling future stories
|
||||
m = MUserStory(story=story, user_id=request.user.pk, feed_id=feed_id, read_date=date)
|
||||
m = MUserStory(story=story, user_id=request.user.pk, feed_id=feed_id, read_date=date, story_date=story.story_date)
|
||||
try:
|
||||
m.save()
|
||||
except OperationError:
|
||||
logging.user(request.user, "~BRMarked story as read: Duplicate Story -> %s" % (story_id))
|
||||
m = MUserStory.objects.get(story=story, user_id=request.user.pk, feed_id=feed_id)
|
||||
m.read_date = date
|
||||
m.story_date = story.story_date
|
||||
m.save()
|
||||
|
||||
return data
|
||||
|
|
@ -684,7 +683,6 @@ def mark_feed_as_read(request):
|
|||
code = 1
|
||||
|
||||
logging.user(request.user, "~FMMarking feed as read: ~SB%s" % (feed,))
|
||||
MUserStory.objects(user_id=request.user.pk, feed_id=feed_id).delete()
|
||||
return dict(code=code)
|
||||
|
||||
def _parse_user_info(user):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue