mirror of
https://github.com/samuelclay/NewsBlur.git
synced 2025-09-18 21:50:56 +00:00
Fixing feed update times for inactive feeds to never go inadctive but to instead go into geometrically longer feed fetches.
This commit is contained in:
Samuel Clay
parent
a538816978
commit
34ad54e1f5
4 changed files with 103 additions and 14 deletions
|
|
@ -34,18 +34,16 @@ class Command(BaseCommand):
|
|||
last_update__lte=day,
|
||||
queued_date__lte=day,
|
||||
min_to_decay__lte=60*24,
|
||||
active_subscribers__gte=1,
|
||||
active=True
|
||||
).order_by('?')
|
||||
active_subscribers__gte=1
|
||||
).order_by('?')[:50]
|
||||
if feeds: Feed.task_feeds(feeds)
|
||||
|
||||
week = now - datetime.timedelta(days=7)
|
||||
feeds = Feed.objects.filter(
|
||||
last_update__lte=week,
|
||||
queued_date__lte=day,
|
||||
active_subscribers__gte=1,
|
||||
active=True
|
||||
).order_by('?')
|
||||
active_subscribers__gte=1
|
||||
).order_by('?')[:50]
|
||||
if feeds: Feed.task_feeds(feeds)
|
||||
|
||||
# feeds = Feed.objects.filter(
|
||||
|
|
|
|||
87
apps/rss_feeds/migrations/0058_errors_since_good.py
Normal file
87
apps/rss_feeds/migrations/0058_errors_since_good.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import datetime
|
||||
from south.db import db
|
||||
from south.v2 import SchemaMigration
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(SchemaMigration):
|
||||
|
||||
def forwards(self, orm):
|
||||
# Adding field 'Feed.errors_since_good'
|
||||
db.add_column('feeds', 'errors_since_good',
|
||||
self.gf('django.db.models.fields.IntegerField')(default=0),
|
||||
keep_default=False)
|
||||
|
||||
|
||||
def backwards(self, orm):
|
||||
# Deleting field 'Feed.errors_since_good'
|
||||
db.delete_column('feeds', 'errors_since_good')
|
||||
|
||||
|
||||
models = {
|
||||
'rss_feeds.duplicatefeed': {
|
||||
'Meta': {'object_name': 'DuplicateFeed'},
|
||||
'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
|
||||
'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}),
|
||||
'duplicate_link': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}),
|
||||
'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
|
||||
},
|
||||
'rss_feeds.feed': {
|
||||
'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
|
||||
'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
|
||||
'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
|
||||
'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
|
||||
'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
|
||||
'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
|
||||
'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
|
||||
'errors_since_good': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
|
||||
'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
|
||||
'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255', 'db_index': 'True'}),
|
||||
'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
|
||||
'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
|
||||
'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
|
||||
'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
|
||||
'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
|
||||
'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
|
||||
'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
|
||||
'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
|
||||
'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
|
||||
'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
|
||||
'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
|
||||
'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
|
||||
'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
|
||||
},
|
||||
'rss_feeds.feeddata': {
|
||||
'Meta': {'object_name': 'FeedData'},
|
||||
'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
|
||||
'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
|
||||
'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
|
||||
'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
|
||||
},
|
||||
'rss_feeds.feedloadtime': {
|
||||
'Meta': {'object_name': 'FeedLoadtime'},
|
||||
'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
|
||||
'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'loadtime': ('django.db.models.fields.FloatField', [], {})
|
||||
}
|
||||
}
|
||||
|
||||
complete_apps = ['rss_feeds']
|
||||
|
|
@ -63,6 +63,7 @@ class Feed(models.Model):
|
|||
has_page_exception = models.BooleanField(default=False, db_index=True)
|
||||
has_page = models.BooleanField(default=True)
|
||||
exception_code = models.IntegerField(default=0)
|
||||
errors_since_good = models.IntegerField(default=0)
|
||||
min_to_decay = models.IntegerField(default=0)
|
||||
days_to_trim = models.IntegerField(default=90)
|
||||
creation = models.DateField(auto_now_add=True)
|
||||
|
|
@ -364,9 +365,11 @@ class Feed(models.Model):
|
|||
# for history in old_fetch_histories:
|
||||
# history.delete()
|
||||
if status_code not in (200, 304):
|
||||
errors, non_errors = self.count_errors_in_history('feed', status_code)
|
||||
self.set_next_scheduled_update(error_count=len(errors), non_error_count=len(non_errors))
|
||||
elif self.has_feed_exception:
|
||||
self.errors_since_good += 1
|
||||
self.count_errors_in_history('feed', status_code)
|
||||
self.set_next_scheduled_update()
|
||||
elif self.has_feed_exception or self.errors_since_good:
|
||||
self.errors_since_good = 0
|
||||
self.has_feed_exception = False
|
||||
self.active = True
|
||||
self.save()
|
||||
|
|
@ -398,9 +401,10 @@ class Feed(models.Model):
|
|||
errors = [h for h in fetch_history if int(h) not in (200, 304)]
|
||||
|
||||
if len(non_errors) == 0 and len(errors) > 1:
|
||||
self.active = True
|
||||
if exception_type == 'feed':
|
||||
self.has_feed_exception = True
|
||||
self.active = False
|
||||
# self.active = False # No longer, just geometrically fetch
|
||||
elif exception_type == 'page':
|
||||
self.has_page_exception = True
|
||||
self.exception_code = status_code or int(errors[0])
|
||||
|
|
@ -1135,12 +1139,12 @@ class Feed(models.Model):
|
|||
|
||||
return total, random_factor*2
|
||||
|
||||
def set_next_scheduled_update(self, error_count=0, non_error_count=0):
|
||||
def set_next_scheduled_update(self):
|
||||
total, random_factor = self.get_next_scheduled_update(force=True, verbose=False)
|
||||
|
||||
if error_count:
|
||||
total = total * error_count
|
||||
logging.debug(' ---> [%-30s] ~FBScheduling feed fetch geometrically: ~SB%s/%s errors. Time: %s min' % (unicode(self)[:30], error_count, non_error_count, total))
|
||||
if self.errors_since_good:
|
||||
total = total * self.errors_since_good
|
||||
logging.debug(' ---> [%-30s] ~FBScheduling feed fetch geometrically: ~SB%s errors. Time: %s min' % (unicode(self)[:30], self.errors_since_good, total))
|
||||
|
||||
next_scheduled_update = datetime.datetime.utcnow() + datetime.timedelta(
|
||||
minutes = total + random_factor)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue