diff --git a/apps/reader/views.py b/apps/reader/views.py index 99596b7eb..41bee3113 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -114,8 +114,10 @@ def load_feeds(request): 'ps': sub.unread_count_positive, 'nt': sub.unread_count_neutral, 'ng': sub.unread_count_negative, - 'updated': format_relative_date(sub.feed.last_update), + 'updated': format_relative_date(sub.feed.last_update) } + if not sub.feed.fetched_once: + feeds[sub.feed.pk]['not_yet_fetched'] = True data = dict(feeds=feeds, folders=json.decode(folders.folders)) return data @@ -187,6 +189,8 @@ def refresh_feeds(request): 'nt': sub.unread_count_neutral, 'ng': sub.unread_count_negative, } + if request.GET.get('check_fetch_status', False) and not sub.feed.fetched_once: + feeds[sub.feed.pk]['not_yet_fetched'] = True return feeds diff --git a/apps/rss_feeds/migrations/0011_fetched_once.py b/apps/rss_feeds/migrations/0011_fetched_once.py new file mode 100644 index 000000000..06fa26e33 --- /dev/null +++ b/apps/rss_feeds/migrations/0011_fetched_once.py @@ -0,0 +1,125 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'Feed.fetched_once' + db.add_column('feeds', 'fetched_once', self.gf('django.db.models.fields.BooleanField')(default=False, blank=True), keep_default=False) + + + def backwards(self, orm): + + # Deleting field 'Feed.fetched_once' + db.delete_column('feeds', 'fetched_once') + + + models = { + 'rss_feeds.feed': { + 'Meta': {'object_name': 'Feed', 'db_table': "'feeds'"}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}), + 'etag': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'feed_address': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '255'}), + 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}), + 'feed_tagline': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'feed_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '15'}), + 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'stories_last_year': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedfetchhistory': { + 'Meta': {'object_name': 'FeedFetchHistory'}, + 'exception': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'feed_fetch_history'", 'to': "orm['rss_feeds.Feed']"}), + 'fetch_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'status_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedpage': { + 'Meta': {'object_name': 'FeedPage'}, + 'feed': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'feed_page'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'page_data': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedupdatehistory': { + 'Meta': {'object_name': 'FeedUpdateHistory'}, + 'average_per_feed': ('django.db.models.fields.DecimalField', [], {'max_digits': '4', 'decimal_places': '1'}), + 'fetch_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'number_of_feeds': ('django.db.models.fields.IntegerField', [], {}), + 'seconds_taken': ('django.db.models.fields.IntegerField', [], {}) + }, + 'rss_feeds.feedxml': { + 'Meta': {'object_name': 'FeedXML'}, + 'feed': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'feed_xml'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'rss_xml': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.pagefetchhistory': { + 'Meta': {'object_name': 'PageFetchHistory'}, + 'exception': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'page_fetch_history'", 'to': "orm['rss_feeds.Feed']"}), + 'fetch_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'status_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.storiespermonth': { + 'Meta': {'object_name': 'StoriesPerMonth'}, + 'beginning_of_month': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'stories_per_month'", 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'month': ('django.db.models.fields.IntegerField', [], {}), + 'story_count': ('django.db.models.fields.IntegerField', [], {}), + 'year': ('django.db.models.fields.IntegerField', [], {}) + }, + 'rss_feeds.story': { + 'Meta': {'unique_together': "(('story_feed', 'story_guid_hash'),)", 'object_name': 'Story', 'db_table': "'stories'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'story_author': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.StoryAuthor']"}), + 'story_author_name': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'story_content': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}), + 'story_content_type': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'story_date': ('django.db.models.fields.DateTimeField', [], {}), + 'story_feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'stories'", 'to': "orm['rss_feeds.Feed']"}), + 'story_guid': ('django.db.models.fields.CharField', [], {'max_length': '1000'}), + 'story_guid_hash': ('django.db.models.fields.CharField', [], {'max_length': '40'}), + 'story_original_content': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}), + 'story_past_trim_date': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'story_permalink': ('django.db.models.fields.CharField', [], {'max_length': '1000'}), + 'story_tags': ('django.db.models.fields.CharField', [], {'max_length': '2000', 'null': 'True', 'blank': 'True'}), + 'story_title': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['rss_feeds.Tag']", 'symmetrical': 'False'}) + }, + 'rss_feeds.storyauthor': { + 'Meta': {'object_name': 'StoryAuthor'}, + 'author_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'rss_feeds.tag': { + 'Meta': {'object_name': 'Tag'}, + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + } + } + + complete_apps = ['rss_feeds'] diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 2985d9b0c..945846f1b 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -30,7 +30,8 @@ class Feed(models.Model): feed_tagline = models.CharField(max_length=1024, default="", blank=True, null=True) active = models.BooleanField(default=True) num_subscribers = models.IntegerField(default=0) - last_update = models.DateTimeField(auto_now=True, default=0) + last_update = models.DateTimeField(auto_now=True) + fetched_once = models.BooleanField(default=False) min_to_decay = models.IntegerField(default=15) days_to_trim = models.IntegerField(default=90) creation = models.DateField(auto_now_add=True) @@ -657,8 +658,45 @@ class StoriesPerMonth(models.Model): return month_counts, average_per_month @classmethod - def recount_feed(cls, feed): + def recount_feed(cls, feed, current_counts=None): d = defaultdict(int) - stories = Story.objects.filter(story_feed=feed).extra(select={'year': "EXTRACT(year FROM story_date)", 'month': "EXTRACT(month from story_date)"}).values('year', 'month') + now = datetime.datetime.now() + min_year = now.year + if not current_counts: + current_counts = [] + + # Count stories, aggregate by year and month + stories = Story.objects.filter(story_feed=feed).extra(select={ + 'year': "EXTRACT(year FROM story_date)", + 'month': "EXTRACT(month from story_date)" + }).values('year', 'month') for story in stories: - pass \ No newline at end of file + year = int(story['year']) + d['%s-%s' % (year, int(story['month']))] += 1 + if year < min_year: + min_year = year + + # Add on to existing months, always amending up, never down. (Current month + # is guaranteed to be accurate, since trim_feeds won't delete it until after + # a month. Hacker News can have 1,000+ and still be counted.) + for current_month, current_count in current_counts: + if current_month not in d or d[current_month] < current_count: + d[current_month] = current_count + year = re.findall(r"(\d{4})-\d{1,2}", current_month)[0] + if year < min_year: + min_year = year + + # Assemble a list with 0's filled in for missing months, + # trimming left and right 0's. + months = [] + start = False + for year in range(min_year, now.year+1): + for month in range(1, 12+1): + if datetime.datetime(year, month, 1) < now: + key = '%s-%s' % (year, month) + if d.get(key) or start: + start = True + months.append((key, d.get(key, 0))) + from pprint import pprint + pprint(months) + \ No newline at end of file diff --git a/media/css/reader.css b/media/css/reader.css index bad2e2325..1d18012eb 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -303,6 +303,7 @@ a img { .left-pane { display: none; overflow: hidden !important; + background-color: #D7DDE6; } .left-center { @@ -321,13 +322,30 @@ a img { /* = Feed List = */ /* ============= */ +#NB-feeds-list-loader { + background: transparent url("../img/reader/big_spinner.gif") no-repeat 0 0; + color: #C7CDD6; + font-size: 16px; + height: 51px; + left: 5%; + padding: 5px 0 0 62px; + position: absolute; + text-shadow: 0 1px 0 #E7EDF6; + text-transform: uppercase; + top: 40%; + width: 125px; + z-index: 10; + cursor: default; + display: none; +} + #feed_list ::-moz-selection { -background: transparent; -} + background: transparent; +} #feed_list ::selection { -background: transparent; + background: transparent; } #feed_list { diff --git a/media/img/reader/big_spinner.gif b/media/img/reader/big_spinner.gif new file mode 100644 index 000000000..927bab412 Binary files /dev/null and b/media/img/reader/big_spinner.gif differ diff --git a/media/js/newsblur/reader.js b/media/js/newsblur/reader.js index 738556617..e8af63c72 100644 --- a/media/js/newsblur/reader.js +++ b/media/js/newsblur/reader.js @@ -19,7 +19,8 @@ $feed_view: $('.NB-feed-story-view'), $story_iframe: $('.NB-feed-frame'), $intelligence_slider: $('.NB-intelligence-slider'), - $mouse_indicator: $('#mouse-indicator') + $mouse_indicator: $('#mouse-indicator'), + $feed_link_loader: $('#NB-feeds-list-loader') }; this.flags = { 'feed_view_images_loaded': {}, @@ -519,6 +520,7 @@ if ($('#feed_list').length) { $('.NB-callout-ftux .NB-callout-text').text('Loading feeds...'); + this.$s.$feed_link_loader.css({'display': 'block'}); this.model.load_feeds($.rescope(this.make_feeds, this)); } }, @@ -527,11 +529,16 @@ var $feed_list = this.$s.$feed_list.empty(); var folders = this.model.folders; var feeds = this.model.feeds; + // NEWSBLUR.log(['Making feeds', {'folders': folders, 'feeds': feeds}]); $('#story_taskbar').css({'display': 'block'}); + $feed_list.css({'display': 'none'}); this.make_feeds_folder($feed_list, folders); + this.$s.$feed_link_loader.fadeOut(250, function() { + $feed_list.fadeIn(750); + }); if (!folders.length) { this.setup_ftux_add_feed_callout(); @@ -563,7 +570,7 @@ ]); (function($feeds, $folder) { setTimeout(function() { - $feeds.append($folder); + $feeds.append($folder); $('.unread_count', $feeds).corner('4px'); }, 50); })($feeds, $folder); diff --git a/templates/reader/feeds.xhtml b/templates/reader/feeds.xhtml index 03d90074a..550af0fa2 100644 --- a/templates/reader/feeds.xhtml +++ b/templates/reader/feeds.xhtml @@ -263,6 +263,7 @@
+
Everything is on its way...
diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index 3270c1000..732eaf007 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -307,6 +307,7 @@ class Dispatcher: comment = u'' feed.last_load_time = max(1, delta.seconds) + feed.fetched_once = True try: feed.save() except IntegrityError: