diff --git a/apps/profile/models.py b/apps/profile/models.py index cd00bd2e2..75a68a0aa 100644 --- a/apps/profile/models.py +++ b/apps/profile/models.py @@ -1074,28 +1074,28 @@ class RNewUserQueue: @classmethod def add_user(cls, user_id): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) now = time.time() r.zadd(cls.KEY, user_id, now) @classmethod def user_count(cls): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) count = r.zcard(cls.KEY) return count @classmethod def user_position(cls, user_id): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) position = r.zrank(cls.KEY, user_id) if position >= 0: return position + 1 @classmethod def pop_user(cls): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) user = r.zrange(cls.KEY, 0, 0)[0] r.zrem(cls.KEY, user) diff --git a/apps/reader/models.py b/apps/reader/models.py index 3062514a8..b3987fd54 100644 --- a/apps/reader/models.py +++ b/apps/reader/models.py @@ -847,7 +847,7 @@ class UserSubscription(models.Model): @classmethod def verify_feeds_scheduled(cls, user_id): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) user = User.objects.get(pk=user_id) subs = cls.objects.filter(user=user) feed_ids = [sub.feed.pk for sub in subs] diff --git a/apps/rss_feeds/management/shellplus_imports.py b/apps/rss_feeds/management/shellplus_imports.py index 3c5da5c76..6b8bdc1f3 100644 --- a/apps/rss_feeds/management/shellplus_imports.py +++ b/apps/rss_feeds/management/shellplus_imports.py @@ -3,5 +3,5 @@ import redis from apps.social.models import * -r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) +r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) print "Redis: %s" % r \ No newline at end of file diff --git a/apps/rss_feeds/migrations/0067_feed_next_update_redis.py b/apps/rss_feeds/migrations/0067_feed_next_update_redis.py index e205776c5..22399b433 100644 --- a/apps/rss_feeds/migrations/0067_feed_next_update_redis.py +++ b/apps/rss_feeds/migrations/0067_feed_next_update_redis.py @@ -7,7 +7,7 @@ from apps.rss_feeds.models import Feed class Migration(DataMigration): def forwards(self, orm): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) start = 0 for f in xrange(start, Feed.objects.latest('pk').pk, 1000): print " ---> %s" % f diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index f9086f30a..b83c32aa3 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -427,7 +427,7 @@ class Feed(models.Model): @classmethod def task_feeds(cls, feeds, queue_size=12, verbose=True): if not feeds: return - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) if isinstance(feeds, Feed): if verbose: @@ -452,7 +452,7 @@ class Feed(models.Model): @classmethod def drain_task_feeds(cls): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) tasked_feeds = r.zrange('tasked_feeds', 0, -1) logging.debug(" ---> ~FRDraining %s tasked feeds..." % len(tasked_feeds)) @@ -903,7 +903,7 @@ class Feed(models.Model): def update(self, **kwargs): from utils import feed_fetcher - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) original_feed_id = int(self.pk) if getattr(settings, 'TEST_DEBUG', False): @@ -1551,7 +1551,7 @@ class Feed(models.Model): return total def set_next_scheduled_update(self, verbose=False, skip_scheduling=False): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) total = self.get_next_scheduled_update(force=True, verbose=verbose) error_count = self.error_count @@ -1585,13 +1585,13 @@ class Feed(models.Model): @property def error_count(self): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) fetch_errors = int(r.zscore('error_feeds', self.pk) or 0) return fetch_errors + self.errors_since_good def schedule_feed_fetch_immediately(self, verbose=True): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) if verbose: logging.debug(' ---> [%-30s] Scheduling feed fetch immediately...' % (unicode(self)[:30])) @@ -1611,7 +1611,7 @@ class Feed(models.Model): self.save() def queue_pushed_feed_xml(self, xml, latest_push_date_delta=None): - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) queue_size = r.llen("push_feeds") if latest_push_date_delta: diff --git a/apps/rss_feeds/tasks.py b/apps/rss_feeds/tasks.py index 77e5ad7bd..1321582f6 100644 --- a/apps/rss_feeds/tasks.py +++ b/apps/rss_feeds/tasks.py @@ -18,7 +18,7 @@ class TaskFeeds(Task): settings.LOG_TO_STREAM = True now = datetime.datetime.utcnow() start = time.time() - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) tasked_feeds_size = r.zcard('tasked_feeds') hour_ago = now - datetime.timedelta(hours=1) @@ -60,7 +60,7 @@ class TaskBrokenFeeds(Task): settings.LOG_TO_STREAM = True now = datetime.datetime.utcnow() start = time.time() - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) logging.debug(" ---> ~SN~FBQueuing broken feeds...") @@ -126,7 +126,7 @@ class UpdateFeeds(Task): def run(self, feed_pks, **kwargs): from apps.rss_feeds.models import Feed from apps.statistics.models import MStatistics - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0)) compute_scores = bool(mongodb_replication_lag < 10) diff --git a/settings.py b/settings.py index 97b3a6e7a..cf46210f8 100644 --- a/settings.py +++ b/settings.py @@ -652,13 +652,14 @@ CACHES = { REDIS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=0) REDIS_ANALYTICS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=2) REDIS_STATISTICS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=3) -REDIS_FEED_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=4) -REDIS_SESSION_POOL = redis.ConnectionPool(host=SESSION_REDIS_HOST, port=6379, db=5) -# REDIS_CACHE_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=6) # Duped in CACHES -REDIS_PUBSUB_POOL = redis.ConnectionPool(host=REDIS_PUBSUB['host'], port=6379, db=0) -REDIS_STORY_HASH_POOL = redis.ConnectionPool(host=REDIS_STORY['host'], port=6379, db=1) +REDIS_FEED_UPDATE_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=4) # REDIS_STORY_HASH_POOL2 = redis.ConnectionPool(host=REDIS['host'], port=6379, db=8) REDIS_STORY_HASH_TEMP_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=10) +# REDIS_CACHE_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=6) # Duped in CACHES +REDIS_SESSION_POOL = redis.ConnectionPool(host=SESSION_REDIS_HOST, port=6379, db=5) +REDIS_STORY_HASH_POOL = redis.ConnectionPool(host=REDIS_STORY['host'], port=6379, db=1) +REDIS_FEED_SUB_POOL = redis.ConnectionPool(host=REDIS_STORY['host'], port=6379, db=2) +REDIS_PUBSUB_POOL = redis.ConnectionPool(host=REDIS_PUBSUB['host'], port=6379, db=0) # ========== # = Assets = diff --git a/utils/munin/newsblur_feed_counts.py b/utils/munin/newsblur_feed_counts.py index 27e3b5ac7..ee5b8abc3 100755 --- a/utils/munin/newsblur_feed_counts.py +++ b/utils/munin/newsblur_feed_counts.py @@ -50,7 +50,7 @@ class NBMuninGraph(MuninGraph): push_feeds = PushSubscription.objects.filter(verified=True).count() MStatistics.set('munin:push_feeds', push_feeds, 60*60*12) - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) return { 'scheduled_feeds': r.zcard('scheduled_updates'), diff --git a/utils/munin/newsblur_updates.py b/utils/munin/newsblur_updates.py index b1fe29377..cbb14f2b6 100755 --- a/utils/munin/newsblur_updates.py +++ b/utils/munin/newsblur_updates.py @@ -26,7 +26,7 @@ class NBMuninGraph(MuninGraph): def calculate_metrics(self): from django.conf import settings - r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL) + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) return { 'update_queue': r.scard("queued_feeds"),