Adding subscription callbacks for PuSH.

apps/push/migrations/0001_initial.py

@@ -0,0 +1,79 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'PushSubscription'
+        db.create_table('push_pushsubscription', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('feed', self.gf('django.db.models.fields.related.OneToOneField')(related_name='push', unique=True, to=orm['rss_feeds.Feed'])),
+            ('hub', self.gf('django.db.models.fields.URLField')(max_length=200, db_index=True)),
+            ('topic', self.gf('django.db.models.fields.URLField')(max_length=200, db_index=True)),
+            ('verified', self.gf('django.db.models.fields.BooleanField')(default=False)),
+            ('verify_token', self.gf('django.db.models.fields.CharField')(max_length=60)),
+            ('lease_expires', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)),
+        ))
+        db.send_create_signal('push', ['PushSubscription'])
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'PushSubscription'
+        db.delete_table('push_pushsubscription')
+
+
+    models = {
+        'push.pushsubscription': {
+            'Meta': {'object_name': 'PushSubscription'},
+            'feed': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'push'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
+            'hub': ('django.db.models.fields.URLField', [], {'max_length': '200', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lease_expires': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'topic': ('django.db.models.fields.URLField', [], {'max_length': '200', 'db_index': 'True'}),
+            'verified': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'verify_token': ('django.db.models.fields.CharField', [], {'max_length': '60'})
+        },
+        'rss_feeds.feed': {
+            'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+            'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+            'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
+            'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
+            'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255'}),
+            'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
+            'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        }
+    }
+
+    complete_apps = ['push']

apps/push/models.py

@@ -11,13 +11,14 @@ from django.core.urlresolvers import reverse, Resolver404
 from django.db import models
 from django.utils.hashcompat import sha_constructor
 
-from djpubsubhubbub import signals
+from apps.push import signals
+from apps.rss_feeds.models import Feed
 
 DEFAULT_LEASE_SECONDS = 2592000 # 30 days in seconds
 
-class SubscriptionManager(models.Manager):
+class PushSubscriptionManager(models.Manager):
 
-    def subscribe(self, topic, hub=None, callback=None,
+    def subscribe(self, topic, feed, hub=None, callback=None,
                   lease_seconds=None):
         if hub is None:
             hub = self._get_hub(topic)
@@ -31,7 +32,7 @@ class SubscriptionManager(models.Manager):
                                    DEFAULT_LEASE_SECONDS)
 
         subscription, created = self.get_or_create(
-            hub=hub, topic=topic)
+            hub=hub, topic=topic, feed=feed)
         signals.pre_subscribe.send(sender=subscription, created=created)
         subscription.set_expiration(lease_seconds)
 
@@ -43,7 +44,8 @@ class SubscriptionManager(models.Manager):
                 raise TypeError(
                     'callback cannot be None if there is not a reverable URL')
             else:
-                callback = 'http://' + Site.objects.get_current() + \
+                # callback = 'http://' + Site.objects.get_current() + \
+                callback = 'http://' + "dev.newsblur.com" + \
                     callback_path
 
         response = self._send_request(hub, {
@@ -66,6 +68,8 @@ class SubscriptionManager(models.Manager):
                     topic, hub, error))
 
         subscription.save()
+        feed.is_push = subscription.verified
+        feed.save()
         if subscription.verified:
             signals.verified.send(sender=subscription)
         return subscription
@@ -89,15 +93,15 @@ class SubscriptionManager(models.Manager):
         encoded_data = urlencode(list(data_generator()))
         return urllib2.urlopen(url, encoded_data)
 
-class Subscription(models.Model):
+class PushSubscription(models.Model):
-
+    feed = models.OneToOneField(Feed, db_index=True, related_name='push')
-    hub = models.URLField()
+    hub = models.URLField(db_index=True)
-    topic = models.URLField()
+    topic = models.URLField(db_index=True)
     verified = models.BooleanField(default=False)
     verify_token = models.CharField(max_length=60)
     lease_expires = models.DateTimeField(default=datetime.now)
 
-    objects = SubscriptionManager()
+    objects = PushSubscriptionManager()
 
     # class Meta:
     #     unique_together = [

apps/push/tests.py

@@ -48,7 +48,7 @@ class MockResponse(object):
 
 class PSHBTestBase:
 
-    urls = 'djpubsubhubbub.urls'
+    urls = 'apps.djpubsubhubbub.urls'
 
     def setUp(self):
         self._old_send_request = SubscriptionManager._send_request

apps/push/views.py

@@ -6,8 +6,8 @@ import feedparser
 from django.http import HttpResponse, Http404
 from django.shortcuts import get_object_or_404
 
-from djpubsubhubbub.models import Subscription
+from apps.push.models import PushSubscription
-from djpubsubhubbub.signals import verified, updated
+from apps.push.signals import verified, updated
 
 def callback(request, pk):
     if request.method == 'GET':
@@ -20,7 +20,7 @@ def callback(request, pk):
         if mode == 'subscribe':
             if not verify_token.startswith('subscribe'):
                 raise Http404
-            subscription = get_object_or_404(Subscription,
+            subscription = get_object_or_404(PushSubscription,
                                              pk=pk,
                                              topic=topic,
                                              verify_token=verify_token)
@@ -30,7 +30,7 @@ def callback(request, pk):
 
         return HttpResponse(challenge, content_type='text/plain')
     elif request.method == 'POST':
-        subscription = get_object_or_404(Subscription, pk=pk)
+        subscription = get_object_or_404(PushSubscription, pk=pk)
         parsed = feedparser.parse(request.raw_post_data)
         if parsed.feed.links: # single notification
             hub_url = subscription.hub
@@ -52,8 +52,8 @@ def callback(request, pk):
             if needs_update:
                 expiration_time = subscription.lease_expires - datetime.now()
                 seconds = expiration_time.days*86400 + expiration_time.seconds
-                Subscription.objects.subscribe(
+                PushSubscription.objects.subscribe(
-                    self_url, hub_url,
+                    self_url, hub_url, feed=subscription.feed,
                     callback=request.build_absolute_uri(),
                     lease_seconds=seconds)
 

apps/rss_feeds/migrations/0054_push.py

@@ -0,0 +1,84 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding field 'Feed.is_push'
+        db.add_column('feeds', 'is_push', self.gf('django.db.models.fields.NullBooleanField')(default=False, null=True, blank=True), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Deleting field 'Feed.is_push'
+        db.delete_column('feeds', 'is_push')
+
+
+    models = {
+        'rss_feeds.duplicatefeed': {
+            'Meta': {'object_name': 'DuplicateFeed'},
+            'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
+            'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'rss_feeds.feed': {
+            'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+            'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+            'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
+            'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
+            'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255'}),
+            'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
+            'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'rss_feeds.feeddata': {
+            'Meta': {'object_name': 'FeedData'},
+            'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
+            'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+            'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'rss_feeds.feedloadtime': {
+            'Meta': {'object_name': 'FeedLoadtime'},
+            'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'loadtime': ('django.db.models.fields.FloatField', [], {})
+        }
+    }
+
+    complete_apps = ['rss_feeds']

apps/rss_feeds/models.py

@@ -40,6 +40,7 @@ class Feed(models.Model):
     feed_link_locked = models.BooleanField(default=False)
     hash_address_and_link = models.CharField(max_length=64, unique=True, db_index=True)
     feed_title = models.CharField(max_length=255, default="[Untitled]", blank=True, null=True)
+    is_push = models.NullBooleanField(default=False, blank=True, null=True)
     active = models.BooleanField(default=True, db_index=True)
     num_subscribers = models.IntegerField(default=-1)
     active_subscribers = models.IntegerField(default=-1, db_index=True)

settings.py

@@ -209,6 +209,7 @@ INSTALLED_APPS = (
     'apps.statistics',
     'apps.static',
     'apps.mobile',
+    'apps.push',
     'south',
     'utils',
     'vendor',

utils/feed_fetcher.py

@@ -13,6 +13,7 @@ from apps.reader.models import UserSubscription, MUserStory
 from apps.rss_feeds.models import Feed, MStory
 from apps.rss_feeds.page_importer import PageImporter
 from apps.rss_feeds.icon_importer import IconImporter
+from apps.push.models import PushSubscription
 from utils import feedparser
 from utils.story_functions import pre_process_story
 from utils import log as logging
@@ -124,6 +125,17 @@ class ProcessFeed:
                                   unicode(self.feed)[:30],
                                   self.fpf.bozo_exception,
                                   len(self.fpf.entries)))
+            if not self.feed.is_push and self.fpf.feed.links:
+                hub_url = None
+                self_url = self.feed.feed_link
+                for link in parsed.feed.links:
+                    if link['rel'] == 'hub':
+                        hub_url = link['href']
+                    elif link['rel'] == 'self':
+                        self_url = link['href']
+                if hub_url and self_url:
+                    PushSubscription.objects.subscribe(self_url, hub_url, self.feed)
+                    
             if self.fpf.status == 304:
                 self.feed.save()
                 self.feed.save_feed_history(304, "Not modified")