public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-09-18 21:43:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Removing unused redis story objectid db.

Browse source
This commit is contained in:
Samuel Clay 2013-05-02 12:27:37 -07:00
parent 0ff3d41c09
commit 009010f879
5 changed files with 36 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
apps/reader/models.py
View file

@ -95,8 +95,7 @@ class UserSubscription(models.Model):
sub.sync_redis(skip_feed=skip_feed)
def sync_redis(self, skip_feed=False):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD+1)
userstories = MUserStory.objects.filter(feed_id=self.feed_id, user_id=self.user_id,
@ -105,11 +104,9 @@ class UserSubscription(models.Model):
logging.debug(" ---> ~SN~FMSyncing ~SB%s~SN stories (%s)" % (total, self))
pipeline = r.pipeline()
hashpipe = h.pipeline()
for userstory in userstories:
userstory.sync_redis(pipeline=pipeline, hashpipe=hashpipe)
userstory.sync_redis(r=pipeline)
pipeline.execute()
hashpipe.execute()
def get_stories(self, offset=0, limit=6, order='newest', read_filter='all', withscores=False):
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
@ -671,41 +668,28 @@ class MUserStory(mongo.Document):
return story.id
def sync_redis(self, r=None, pipeline=None, hashpipe=None):
if pipeline:
r = pipeline
if hashpipe:
h = pipeline
elif not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
def sync_redis(self, r=None):
if not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
if self.story_db_id:
all_read_stories_key = 'RS:%s' % (self.user_id)
r.sadd(all_read_stories_key, self.story_db_id)
r.sadd(all_read_stories_key, self.feed_guid_hash)
r.expire(all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
h.sadd(all_read_stories_key, self.feed_guid_hash)
h.expire(all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
read_story_key = 'RS:%s:%s' % (self.user_id, self.feed_id)
r.sadd(read_story_key, self.story_db_id)
r.sadd(read_story_key, self.feed_guid_hash)
r.expire(read_story_key, settings.DAYS_OF_UNREAD*24*60*60)
h.sadd(read_story_key, self.feed_guid_hash)
h.expire(read_story_key, settings.DAYS_OF_UNREAD*24*60*60)
def remove_from_redis(self):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
if self.story_db_id:
r.srem('RS:%s' % self.user_id, self.story_db_id)
r.srem('RS:%s:%s' % (self.user_id, self.feed_id), self.story_db_id)
h.srem('RS:%s' % self.user_id, self.feed_guid_hash)
h.srem('RS:%s:%s' % (self.user_id, self.feed_id), self.feed_guid_hash)
r.srem('RS:%s' % self.user_id, self.feed_guid_hash)
r.srem('RS:%s:%s' % (self.user_id, self.feed_id), self.feed_guid_hash)
@classmethod
def sync_all_redis(cls, user_id=None, feed_id=None, force=False):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD+1)
if feed_id and user_id:
@ -714,17 +698,12 @@ class MUserStory(mongo.Document):
read_date__gte=UNREAD_CUTOFF)
key = "RS:%s:%s" % (user_id, feed_id)
r.delete(key)
h.delete(key)
elif feed_id:
read_stories = cls.objects.filter(feed_id=feed_id, read_date__gte=UNREAD_CUTOFF)
keys = r.keys("RS:*:%s" % feed_id)
print " ---> Deleting %s redis keys: %s" % (len(keys), keys)
for key in keys:
r.delete(key)
keys = h.keys("RS:*:%s" % feed_id)
print " ---> Deleting %s redis keys: %s" % (len(keys), keys)
for key in keys:
h.delete(key)
elif user_id:
read_stories = cls.objects.filter(user_id=user_id, read_date__gte=UNREAD_CUTOFF)
keys = r.keys("RS:%s:*" % user_id)
@ -732,11 +711,6 @@ class MUserStory(mongo.Document):
print " ---> Deleting %s redis keys: %s" % (len(keys), keys)
for key in keys:
r.delete(key)
keys = h.keys("RS:%s:*" % user_id)
h.delete("RS:%s" % user_id)
print " ---> Deleting %s redis keys: %s" % (len(keys), keys)
for key in keys:
h.delete(key)
elif force:
read_stories = cls.objects.all(read_date__gte=UNREAD_CUTOFF)
else:
@ -745,21 +719,16 @@ class MUserStory(mongo.Document):
total = read_stories.count()
logging.debug(" ---> ~SN~FMSyncing ~SB%s~SN stories (%s/%s)" % (total, user_id, feed_id))
pipeline = None
hashpipe = None
for i, read_story in enumerate(read_stories):
if not pipeline:
pipeline = r.pipeline()
hashpipe = h.pipeline()
if (i+1) % 1000 == 0:
print " ---> %s/%s" % (i+1, total)
pipeline.execute()
hashpipe.execute()
pipeline = r.pipeline()
hashpipe = h.pipeline()
read_story.sync_redis(r, pipeline=pipeline, hashpipe=hashpipe)
read_story.sync_redis(r=pipeline)
if pipeline:
pipeline.execute()
hashpipe.execute()
class UserSubscriptionFolders(models.Model):
"""

2
apps/reader/views.py
View file

@ -503,7 +503,7 @@ def load_single_feed(request, feed_id):
if query:
stories = feed.find_stories(query, offset=offset, limit=limit)
elif usersub:
elif usersub and (read_filter == 'unread' or order == 'oldest'):
stories = usersub.get_stories(order=order, read_filter=read_filter, offset=offset, limit=limit)
else:
stories = feed.get_stories(offset, limit)

51
apps/rss_feeds/models.py
View file

@ -227,7 +227,7 @@ class Feed(models.Model):
def sync_redis(self):
return MStory.sync_all_redis(self.pk)
return MStory.sync_feed_redis(self.pk)
def sync_autocompletion(self):
if self.num_subscribers <= 10 and not settings.DEBUG: return
@ -1649,53 +1649,38 @@ class MStory(mongo.Document):
return stories
def sync_redis(self, r=None, h=None):
def sync_redis(self, r=None):
if not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
if not h:
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
UNREAD_CUTOFF = datetime.datetime.now() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
if self.id and self.story_date > UNREAD_CUTOFF:
r.sadd('F:%s' % self.story_feed_id, self.id)
r.zadd('zF:%s' % self.story_feed_id, self.id, time.mktime(self.story_date.timetuple()))
r.sadd('F:%s' % self.story_feed_id, self.story_hash)
r.zadd('zF:%s' % self.story_feed_id, self.story_hash, time.mktime(self.story_date.timetuple()))
r.expire('F:%s' % self.story_feed_id, settings.DAYS_OF_UNREAD*24*60*60)
r.expire('zF:%s' % self.story_feed_id, settings.DAYS_OF_UNREAD*24*60*60)
h.sadd('F:%s' % self.story_feed_id, self.story_hash)
h.zadd('zF:%s' % self.story_feed_id, self.story_hash, time.mktime(self.story_date.timetuple()))
h.expire('F:%s' % self.story_feed_id, settings.DAYS_OF_UNREAD*24*60*60)
h.expire('zF:%s' % self.story_feed_id, settings.DAYS_OF_UNREAD*24*60*60)
def remove_from_redis(self, r=None, h=None):
def remove_from_redis(self, r=None):
if not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
if not h:
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
if self.id:
r.srem('F:%s' % self.story_feed_id, self.id)
r.zrem('zF:%s' % self.story_feed_id, self.id)
h.srem('F:%s' % self.story_feed_id, self.story_hash)
h.zrem('zF:%s' % self.story_feed_id, self.story_hash)
r.srem('F:%s' % self.story_feed_id, self.story_hash)
r.zrem('zF:%s' % self.story_feed_id, self.story_hash)
@classmethod
def sync_all_redis(cls, story_feed_id=None):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
def sync_feed_redis(cls, story_feed_id):
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
UNREAD_CUTOFF = datetime.datetime.now() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
feed = None
if story_feed_id:
feed = Feed.get_by_id(story_feed_id)
stories = cls.objects.filter(story_date__gte=UNREAD_CUTOFF)
if story_feed_id:
stories = stories.filter(story_feed_id=story_feed_id)
r.delete('F:%s' % story_feed_id)
r.delete('zF:%s' % story_feed_id)
h.delete('F:%s' % story_feed_id)
h.delete('zF:%s' % story_feed_id)
feed = Feed.get_by_id(story_feed_id)
stories = cls.objects.filter(story_feed_id=story_feed_id, story_date__gte=UNREAD_CUTOFF)
r.delete('F:%s' % story_feed_id)
r.delete('zF:%s' % story_feed_id)
logging.info(" ---> [%-30s] ~FMSyncing ~SB%s~SN stories to redis" % (feed and feed.title[:30] or story_feed_id, stories.count()))
p = r.pipeline()
for story in stories:
story.sync_redis(r=r, h=h)
story.sync_redis(r=p)
p.execute()
def count_comments(self):
from apps.social.models import MSharedStory

21
apps/social/models.py
View file

@ -1484,7 +1484,6 @@ class MSharedStory(mongo.Document):
@classmethod
def sync_all_redis(cls, drop=False):
r = redis.Redis(connection_pool=settings.REDIS_POOL)
s = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
if drop:
for key_name in ["C", "S"]:
@ -1494,7 +1493,7 @@ class MSharedStory(mongo.Document):
r.delete(key)
for story in cls.objects.all():
story.sync_redis_shares(r=r)
story.sync_redis_story(r=s, h=h)
story.sync_redis_story(r=h)
def sync_redis(self):
self.sync_redis_shares()
@ -1512,26 +1511,19 @@ class MSharedStory(mongo.Document):
else:
r.srem(comment_key, self.user_id)
def sync_redis_story(self, r=None, h=None):
def sync_redis_story(self, r=None):
if not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
if not h:
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
if not self.story_db_id:
self.ensure_story_db_id(save=True)
if self.story_db_id:
r.sadd('B:%s' % self.user_id, self.story_db_id)
r.zadd('zB:%s' % self.user_id, self.story_db_id,
r.sadd('B:%s' % self.user_id, self.feed_guid_hash)
r.zadd('zB:%s' % self.user_id, self.feed_guid_hash,
time.mktime(self.shared_date.timetuple()))
r.expire('B:%s' % self.user_id, settings.DAYS_OF_UNREAD*24*60*60)
r.expire('zB:%s' % self.user_id, settings.DAYS_OF_UNREAD*24*60*60)
h.sadd('B:%s' % self.user_id, self.feed_guid_hash)
h.zadd('zB:%s' % self.user_id, self.feed_guid_hash,
time.mktime(self.shared_date.timetuple()))
h.expire('B:%s' % self.user_id, settings.DAYS_OF_UNREAD*24*60*60)
h.expire('zB:%s' % self.user_id, settings.DAYS_OF_UNREAD*24*60*60)
def remove_from_redis(self):
r = redis.Redis(connection_pool=settings.REDIS_POOL)
@ -1541,9 +1533,6 @@ class MSharedStory(mongo.Document):
comment_key = "C:%s:%s" % (self.story_feed_id, self.guid_hash)
r.srem(comment_key, self.user_id)
s = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
s.srem('B:%s' % self.user_id, self.story_db_id)
s.zrem('zB:%s' % self.user_id, self.story_db_id)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
h.srem('B:%s' % self.user_id, self.feed_guid_hash)
h.zrem('zB:%s' % self.user_id, self.feed_guid_hash)

1
settings.py
View file

@ -545,7 +545,6 @@ MONGOANALYTICSDB = connect(MONGO_ANALYTICS_DB.pop('name'), **MONGO_ANALYTICS_DB)
# =========
REDIS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=0)
REDIS_STORY_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=1)
REDIS_ANALYTICS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=2)
REDIS_STATISTICS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=3)
REDIS_FEED_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=4)