diff --git a/ansible/playbooks/deploy_app.yml b/ansible/playbooks/deploy_app.yml index 98ce0573a..32b20540b 100644 --- a/ansible/playbooks/deploy_app.yml +++ b/ansible/playbooks/deploy_app.yml @@ -65,7 +65,7 @@ git: repo: https://github.com/samuelclay/NewsBlur.git dest: /srv/newsblur/ - version: pipeline + version: pro register: pulled tags: - static diff --git a/ansible/playbooks/deploy_staging.yml b/ansible/playbooks/deploy_staging.yml index 5383acae1..a2c906d0d 100644 --- a/ansible/playbooks/deploy_staging.yml +++ b/ansible/playbooks/deploy_staging.yml @@ -10,7 +10,7 @@ git: repo: https://github.com/samuelclay/NewsBlur.git dest: /srv/newsblur/ - version: master + version: pro register: pulled - name: Reload gunicorn diff --git a/apps/profile/models.py b/apps/profile/models.py index 4f848d02b..906cd4cfa 100644 --- a/apps/profile/models.py +++ b/apps/profile/models.py @@ -642,8 +642,10 @@ class Profile(models.Model): premium = 0 active = 0 active_premium = 0 + pro = 0 key = 's:%s' % feed_id premium_key = 'sp:%s' % feed_id + pro_key = 'spro:%s' % feed_id if user_id: active = UserSubscription.objects.get(feed_id=feed_id, user_id=user_id).only('active').active @@ -651,12 +653,13 @@ class Profile(models.Model): else: user_ids = dict([(us.user_id, us.active) for us in UserSubscription.objects.filter(feed_id=feed_id).only('user', 'active')]) - profiles = Profile.objects.filter(user_id__in=list(user_ids.keys())).values('user_id', 'last_seen_on', 'is_premium') + profiles = Profile.objects.filter(user_id__in=list(user_ids.keys())).values('user_id', 'last_seen_on', 'is_premium', 'is_pro') feed = Feed.get_by_id(feed_id) if entire_feed_counted: r.delete(key) r.delete(premium_key) + r.delete(pro_key) for profiles_group in chunks(profiles, 20): pipeline = r.pipeline() @@ -672,6 +675,11 @@ class Profile(models.Model): premium += 1 else: pipeline.zrem(premium_key, profile['user_id']) + if profile['is_pro']: + pipeline.zadd(pro_key, { profile['user_id']: last_seen_on }) + pro += 1 + else: + pipeline.zrem(pro_key, profile['user_id']) if profile['last_seen_on'] > SUBSCRIBER_EXPIRE and not muted_feed: active += 1 if profile['is_premium']: @@ -686,8 +694,8 @@ class Profile(models.Model): r.zadd(premium_key, {-1: now}) r.expire(premium_key, settings.SUBSCRIBER_EXPIRE*24*60*60) - logging.info(" ---> [%-30s] ~SN~FBCounting subscribers, storing in ~SBredis~SN: ~FMt:~SB~FM%s~SN a:~SB%s~SN p:~SB%s~SN ap:~SB%s" % - (feed.log_title[:30], total, active, premium, active_premium)) + logging.info(" ---> [%-30s] ~SN~FBCounting subscribers, storing in ~SBredis~SN: ~FMt:~SB~FM%s~SN a:~SB%s~SN p:~SB%s~SN ap:~SB%s~SN pro:~SB%s" % + (feed.log_title[:30], total, active, premium, active_premium, pro)) @classmethod def count_all_feed_subscribers_for_user(self, user): @@ -705,6 +713,7 @@ class Profile(models.Model): for feed_id in feeds_group: key = 's:%s' % feed_id premium_key = 'sp:%s' % feed_id + pro_key = 'spro:%s' % feed_id last_seen_on = int(user.profile.last_seen_on.strftime('%s')) if feed_ids is muted_feed_ids: @@ -714,6 +723,10 @@ class Profile(models.Model): pipeline.zadd(premium_key, { user.pk: last_seen_on }) else: pipeline.zrem(premium_key, user.pk) + if user.profile.is_pro: + pipeline.zadd(pro_key, { user.pk: last_seen_on }) + else: + pipeline.zrem(pro_key, user.pk) pipeline.execute() def send_new_user_email(self): diff --git a/apps/push/views.py b/apps/push/views.py index 16a1c43a7..998762705 100644 --- a/apps/push/views.py +++ b/apps/push/views.py @@ -57,7 +57,7 @@ def push_callback(request, push_id): # Don't give fat ping, just fetch. # subscription.feed.queue_pushed_feed_xml(request.body) - if subscription.feed.active_premium_subscribers >= 1: + if subscription.feed.active_subscribers >= 1: subscription.feed.queue_pushed_feed_xml("Fetch me", latest_push_date_delta=latest_push_date_delta) MFetchHistory.add(feed_id=subscription.feed_id, fetch_type='push') diff --git a/apps/rss_feeds/migrations/0004_feed_pro_subscribers.py b/apps/rss_feeds/migrations/0004_feed_pro_subscribers.py new file mode 100644 index 000000000..7579e56ff --- /dev/null +++ b/apps/rss_feeds/migrations/0004_feed_pro_subscribers.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.10 on 2022-01-10 21:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rss_feeds', '0003_auto_20220110_2105'), + ] + + operations = [ + migrations.AddField( + model_name='feed', + name='pro_subscribers', + field=models.IntegerField(blank=True, default=0, null=True), + ), + ] diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 0822c5c1f..884784ca8 100755 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -65,6 +65,7 @@ class Feed(models.Model): num_subscribers = models.IntegerField(default=-1) active_subscribers = models.IntegerField(default=-1, db_index=True) premium_subscribers = models.IntegerField(default=-1) + pro_subscribers = models.IntegerField(default=0, null=True, blank=True) active_premium_subscribers = models.IntegerField(default=-1) branch_from_feed = models.ForeignKey('Feed', blank=True, null=True, db_index=True, on_delete=models.CASCADE) last_update = models.DateTimeField(db_index=True) @@ -100,13 +101,14 @@ class Feed(models.Model): if not self.feed_title: self.feed_title = "[Untitled]" self.save() - return "%s%s: %s - %s/%s/%s" % ( + return "%s%s: %s - %s/%s/%s/%s" % ( self.pk, (" [B: %s]" % self.branch_from_feed.pk if self.branch_from_feed else ""), self.feed_title, self.num_subscribers, self.active_subscribers, self.active_premium_subscribers, + self.pro_subscribers, ) @property @@ -149,6 +151,8 @@ class Feed(models.Model): @property def unread_cutoff(self): + if self.pro_subscribers > 0: + return datetime.datetime.utcnow() - datetime.timedelta(days=9999) if self.active_premium_subscribers > 0: return datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD) @@ -816,6 +820,7 @@ class Feed(models.Model): total = 0 active = 0 premium = 0 + pro = 0 active_premium = 0 # Include all branched feeds in counts @@ -831,10 +836,12 @@ class Feed(models.Model): # now+1 ensures `-1` flag will be corrected for later with - 1 total_key = "s:%s" % feed_id premium_key = "sp:%s" % feed_id + pro_key = "spro:%s" % feed_id pipeline.zcard(total_key) pipeline.zcount(total_key, subscriber_expire, now+1) pipeline.zcard(premium_key) pipeline.zcount(premium_key, subscriber_expire, now+1) + pipeline.zcard(pro_key) results = pipeline.execute() @@ -843,13 +850,15 @@ class Feed(models.Model): active += max(0, results[1] - 1) premium += max(0, results[2] - 1) active_premium += max(0, results[3] - 1) + pro += max(0, results[4] - 1) original_num_subscribers = self.num_subscribers original_active_subs = self.active_subscribers original_premium_subscribers = self.premium_subscribers original_active_premium_subscribers = self.active_premium_subscribers - logging.info(" ---> [%-30s] ~SN~FBCounting subscribers from ~FCredis~FB: ~FMt:~SB~FM%s~SN a:~SB%s~SN p:~SB%s~SN ap:~SB%s ~SN~FC%s" % - (self.log_title[:30], total, active, premium, active_premium, "(%s branches)" % (len(feed_ids)-1) if len(feed_ids)>1 else "")) + original_pro_subscribers = self.pro_subscribers + logging.info(" ---> [%-30s] ~SN~FBCounting subscribers from ~FCredis~FB: ~FMt:~SB~FM%s~SN a:~SB%s~SN p:~SB%s~SN ap:~SB%s pro:~SB%s ~SN~FC%s" % + (self.log_title[:30], total, active, premium, active_premium, pro, "(%s branches)" % (len(feed_ids)-1) if len(feed_ids)>1 else "")) else: from apps.reader.models import UserSubscription @@ -872,6 +881,14 @@ class Feed(models.Model): ) original_premium_subscribers = self.premium_subscribers premium = premium_subs.count() + + pro_subs = UserSubscription.objects.filter( + feed__in=feed_ids, + active=True, + user__profile__is_pro=True + ) + original_pro_subscribers = self.pro_subscribers + pro = pro_subs.count() active_premium_subscribers = UserSubscription.objects.filter( feed__in=feed_ids, @@ -881,8 +898,8 @@ class Feed(models.Model): ) original_active_premium_subscribers = self.active_premium_subscribers active_premium = active_premium_subscribers.count() - logging.debug(" ---> [%-30s] ~SN~FBCounting subscribers from ~FYpostgres~FB: ~FMt:~SB~FM%s~SN a:~SB%s~SN p:~SB%s~SN ap:~SB%s" % - (self.log_title[:30], total, active, premium, active_premium)) + logging.debug(" ---> [%-30s] ~SN~FBCounting subscribers from ~FYpostgres~FB: ~FMt:~SB~FM%s~SN a:~SB%s~SN p:~SB%s~SN ap:~SB%s~SN pro:~SB%s" % + (self.log_title[:30], total, active, premium, active_premium, pro)) if settings.DOCKERBUILD: # Local installs enjoy 100% active feeds @@ -893,15 +910,18 @@ class Feed(models.Model): self.active_subscribers = active self.premium_subscribers = premium self.active_premium_subscribers = active_premium + self.pro_subscribers = pro if (self.num_subscribers != original_num_subscribers or self.active_subscribers != original_active_subs or self.premium_subscribers != original_premium_subscribers or self.active_premium_subscribers != original_active_premium_subscribers): + self.pro_subscribers != original_pro_subscribers): if original_premium_subscribers == -1 or original_active_premium_subscribers == -1: self.save() else: self.save(update_fields=['num_subscribers', 'active_subscribers', - 'premium_subscribers', 'active_premium_subscribers']) + 'premium_subscribers', 'active_premium_subscribers', + 'pro_subscribers']) if verbose: if self.num_subscribers <= 1: @@ -1501,6 +1521,9 @@ class Feed(models.Model): @property def story_cutoff(self): + if self.pro_subscribers >= 1: + return 10000 + cutoff = 500 if self.active_subscribers <= 0: cutoff = 25 @@ -2116,12 +2139,12 @@ class Feed(models.Model): # print 'New/updated story: %s' % (story), return story_in_system, story_has_changed - def get_next_scheduled_update(self, force=False, verbose=True, premium_speed=False): + def get_next_scheduled_update(self, force=False, verbose=True, premium_speed=False, pro_speed=False): if self.min_to_decay and not force and not premium_speed: return self.min_to_decay from apps.notifications.models import MUserFeedNotification - + if premium_speed: self.active_premium_subscribers += 1 @@ -2204,13 +2227,18 @@ class Feed(models.Model): # Twitter feeds get 2 hours minimum if 'twitter' in self.feed_address: total = max(total, 60*2) - + + # Pro subscribers get absolute minimum + if pro_speed or self.pro_subscribers >= 1: + total = min(total, 5) + if verbose: - logging.debug(" ---> [%-30s] Fetched every %s min - Subs: %s/%s/%s Stories/day: %s" % ( + logging.debug(" ---> [%-30s] Fetched every %s min - Subs: %s/%s/%s/%s Stories/day: %s" % ( self.log_title[:30], total, self.num_subscribers, self.active_subscribers, self.active_premium_subscribers, + self.pro_subscribers, spd)) return total