diff --git a/apps/push/migrations/0001_initial.py b/apps/push/migrations/0001_initial.py new file mode 100644 index 000000000..d21150c40 --- /dev/null +++ b/apps/push/migrations/0001_initial.py @@ -0,0 +1,79 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'PushSubscription' + db.create_table('push_pushsubscription', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('feed', self.gf('django.db.models.fields.related.OneToOneField')(related_name='push', unique=True, to=orm['rss_feeds.Feed'])), + ('hub', self.gf('django.db.models.fields.URLField')(max_length=200, db_index=True)), + ('topic', self.gf('django.db.models.fields.URLField')(max_length=200, db_index=True)), + ('verified', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('verify_token', self.gf('django.db.models.fields.CharField')(max_length=60)), + ('lease_expires', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)), + )) + db.send_create_signal('push', ['PushSubscription']) + + + def backwards(self, orm): + + # Deleting model 'PushSubscription' + db.delete_table('push_pushsubscription') + + + models = { + 'push.pushsubscription': { + 'Meta': {'object_name': 'PushSubscription'}, + 'feed': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'push'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'hub': ('django.db.models.fields.URLField', [], {'max_length': '200', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lease_expires': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'topic': ('django.db.models.fields.URLField', [], {'max_length': '200', 'db_index': 'True'}), + 'verified': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'verify_token': ('django.db.models.fields.CharField', [], {'max_length': '60'}) + }, + 'rss_feeds.feed': { + 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), + 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), + 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}), + 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}), + 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}), + 'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255'}), + 'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}), + 'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + } + } + + complete_apps = ['push'] diff --git a/apps/push/migrations/__init__.py b/apps/push/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/push/models.py b/apps/push/models.py index 65a68ed1e..12e71ebe9 100644 --- a/apps/push/models.py +++ b/apps/push/models.py @@ -11,13 +11,14 @@ from django.core.urlresolvers import reverse, Resolver404 from django.db import models from django.utils.hashcompat import sha_constructor -from djpubsubhubbub import signals +from apps.push import signals +from apps.rss_feeds.models import Feed DEFAULT_LEASE_SECONDS = 2592000 # 30 days in seconds -class SubscriptionManager(models.Manager): +class PushSubscriptionManager(models.Manager): - def subscribe(self, topic, hub=None, callback=None, + def subscribe(self, topic, feed, hub=None, callback=None, lease_seconds=None): if hub is None: hub = self._get_hub(topic) @@ -31,7 +32,7 @@ class SubscriptionManager(models.Manager): DEFAULT_LEASE_SECONDS) subscription, created = self.get_or_create( - hub=hub, topic=topic) + hub=hub, topic=topic, feed=feed) signals.pre_subscribe.send(sender=subscription, created=created) subscription.set_expiration(lease_seconds) @@ -43,7 +44,8 @@ class SubscriptionManager(models.Manager): raise TypeError( 'callback cannot be None if there is not a reverable URL') else: - callback = 'http://' + Site.objects.get_current() + \ + # callback = 'http://' + Site.objects.get_current() + \ + callback = 'http://' + "dev.newsblur.com" + \ callback_path response = self._send_request(hub, { @@ -66,6 +68,8 @@ class SubscriptionManager(models.Manager): topic, hub, error)) subscription.save() + feed.is_push = subscription.verified + feed.save() if subscription.verified: signals.verified.send(sender=subscription) return subscription @@ -89,15 +93,15 @@ class SubscriptionManager(models.Manager): encoded_data = urlencode(list(data_generator())) return urllib2.urlopen(url, encoded_data) -class Subscription(models.Model): - - hub = models.URLField() - topic = models.URLField() +class PushSubscription(models.Model): + feed = models.OneToOneField(Feed, db_index=True, related_name='push') + hub = models.URLField(db_index=True) + topic = models.URLField(db_index=True) verified = models.BooleanField(default=False) verify_token = models.CharField(max_length=60) lease_expires = models.DateTimeField(default=datetime.now) - objects = SubscriptionManager() + objects = PushSubscriptionManager() # class Meta: # unique_together = [ diff --git a/apps/push/tests.py b/apps/push/tests.py index 43d1c8e5c..2005b472e 100644 --- a/apps/push/tests.py +++ b/apps/push/tests.py @@ -48,7 +48,7 @@ class MockResponse(object): class PSHBTestBase: - urls = 'djpubsubhubbub.urls' + urls = 'apps.djpubsubhubbub.urls' def setUp(self): self._old_send_request = SubscriptionManager._send_request diff --git a/apps/push/views.py b/apps/push/views.py index d2b259b6b..35393883b 100644 --- a/apps/push/views.py +++ b/apps/push/views.py @@ -6,8 +6,8 @@ import feedparser from django.http import HttpResponse, Http404 from django.shortcuts import get_object_or_404 -from djpubsubhubbub.models import Subscription -from djpubsubhubbub.signals import verified, updated +from apps.push.models import PushSubscription +from apps.push.signals import verified, updated def callback(request, pk): if request.method == 'GET': @@ -20,7 +20,7 @@ def callback(request, pk): if mode == 'subscribe': if not verify_token.startswith('subscribe'): raise Http404 - subscription = get_object_or_404(Subscription, + subscription = get_object_or_404(PushSubscription, pk=pk, topic=topic, verify_token=verify_token) @@ -30,7 +30,7 @@ def callback(request, pk): return HttpResponse(challenge, content_type='text/plain') elif request.method == 'POST': - subscription = get_object_or_404(Subscription, pk=pk) + subscription = get_object_or_404(PushSubscription, pk=pk) parsed = feedparser.parse(request.raw_post_data) if parsed.feed.links: # single notification hub_url = subscription.hub @@ -52,8 +52,8 @@ def callback(request, pk): if needs_update: expiration_time = subscription.lease_expires - datetime.now() seconds = expiration_time.days*86400 + expiration_time.seconds - Subscription.objects.subscribe( - self_url, hub_url, + PushSubscription.objects.subscribe( + self_url, hub_url, feed=subscription.feed, callback=request.build_absolute_uri(), lease_seconds=seconds) diff --git a/apps/rss_feeds/migrations/0054_push.py b/apps/rss_feeds/migrations/0054_push.py new file mode 100644 index 000000000..6d4447fdf --- /dev/null +++ b/apps/rss_feeds/migrations/0054_push.py @@ -0,0 +1,84 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'Feed.is_push' + db.add_column('feeds', 'is_push', self.gf('django.db.models.fields.NullBooleanField')(default=False, null=True, blank=True), keep_default=False) + + + def backwards(self, orm): + + # Deleting field 'Feed.is_push' + db.delete_column('feeds', 'is_push') + + + models = { + 'rss_feeds.duplicatefeed': { + 'Meta': {'object_name': 'DuplicateFeed'}, + 'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'rss_feeds.feed': { + 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), + 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), + 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}), + 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}), + 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}), + 'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255'}), + 'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}), + 'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'rss_feeds.feeddata': { + 'Meta': {'object_name': 'FeedData'}, + 'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedloadtime': { + 'Meta': {'object_name': 'FeedLoadtime'}, + 'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'loadtime': ('django.db.models.fields.FloatField', [], {}) + } + } + + complete_apps = ['rss_feeds'] diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index fbbd6d5e9..064b264b3 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -40,6 +40,7 @@ class Feed(models.Model): feed_link_locked = models.BooleanField(default=False) hash_address_and_link = models.CharField(max_length=64, unique=True, db_index=True) feed_title = models.CharField(max_length=255, default="[Untitled]", blank=True, null=True) + is_push = models.NullBooleanField(default=False, blank=True, null=True) active = models.BooleanField(default=True, db_index=True) num_subscribers = models.IntegerField(default=-1) active_subscribers = models.IntegerField(default=-1, db_index=True) diff --git a/settings.py b/settings.py index 2d3909414..0919a830d 100644 --- a/settings.py +++ b/settings.py @@ -209,6 +209,7 @@ INSTALLED_APPS = ( 'apps.statistics', 'apps.static', 'apps.mobile', + 'apps.push', 'south', 'utils', 'vendor', diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index 29bd00f41..6e1b216d6 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -13,6 +13,7 @@ from apps.reader.models import UserSubscription, MUserStory from apps.rss_feeds.models import Feed, MStory from apps.rss_feeds.page_importer import PageImporter from apps.rss_feeds.icon_importer import IconImporter +from apps.push.models import PushSubscription from utils import feedparser from utils.story_functions import pre_process_story from utils import log as logging @@ -124,6 +125,17 @@ class ProcessFeed: unicode(self.feed)[:30], self.fpf.bozo_exception, len(self.fpf.entries))) + if not self.feed.is_push and self.fpf.feed.links: + hub_url = None + self_url = self.feed.feed_link + for link in parsed.feed.links: + if link['rel'] == 'hub': + hub_url = link['href'] + elif link['rel'] == 'self': + self_url = link['href'] + if hub_url and self_url: + PushSubscription.objects.subscribe(self_url, hub_url, self.feed) + if self.fpf.status == 304: self.feed.save() self.feed.save_feed_history(304, "Not modified")