From aeda525b55df437f433bed58f96bba9fb8aa0512 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 9 Aug 2010 20:44:36 -0400 Subject: [PATCH] 1) Adding new feed story counter. 2) Adding feed fethed initially status. Useful for displaying percentages of fetched feeds for new users. 3) Fade in with loader on feed list. --- apps/reader/views.py | 6 +- .../rss_feeds/migrations/0011_fetched_once.py | 125 ++++++++++++++++++ apps/rss_feeds/models.py | 46 ++++++- media/css/reader.css | 24 +++- media/img/reader/big_spinner.gif | Bin 0 -> 6494 bytes media/js/newsblur/reader.js | 11 +- templates/reader/feeds.xhtml | 1 + utils/feed_fetcher.py | 1 + 8 files changed, 204 insertions(+), 10 deletions(-) create mode 100644 apps/rss_feeds/migrations/0011_fetched_once.py create mode 100644 media/img/reader/big_spinner.gif diff --git a/apps/reader/views.py b/apps/reader/views.py index 99596b7eb..41bee3113 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -114,8 +114,10 @@ def load_feeds(request): 'ps': sub.unread_count_positive, 'nt': sub.unread_count_neutral, 'ng': sub.unread_count_negative, - 'updated': format_relative_date(sub.feed.last_update), + 'updated': format_relative_date(sub.feed.last_update) } + if not sub.feed.fetched_once: + feeds[sub.feed.pk]['not_yet_fetched'] = True data = dict(feeds=feeds, folders=json.decode(folders.folders)) return data @@ -187,6 +189,8 @@ def refresh_feeds(request): 'nt': sub.unread_count_neutral, 'ng': sub.unread_count_negative, } + if request.GET.get('check_fetch_status', False) and not sub.feed.fetched_once: + feeds[sub.feed.pk]['not_yet_fetched'] = True return feeds diff --git a/apps/rss_feeds/migrations/0011_fetched_once.py b/apps/rss_feeds/migrations/0011_fetched_once.py new file mode 100644 index 000000000..06fa26e33 --- /dev/null +++ b/apps/rss_feeds/migrations/0011_fetched_once.py @@ -0,0 +1,125 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'Feed.fetched_once' + db.add_column('feeds', 'fetched_once', self.gf('django.db.models.fields.BooleanField')(default=False, blank=True), keep_default=False) + + + def backwards(self, orm): + + # Deleting field 'Feed.fetched_once' + db.delete_column('feeds', 'fetched_once') + + + models = { + 'rss_feeds.feed': { + 'Meta': {'object_name': 'Feed', 'db_table': "'feeds'"}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}), + 'etag': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'feed_address': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '255'}), + 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}), + 'feed_tagline': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'feed_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '15'}), + 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'stories_last_year': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedfetchhistory': { + 'Meta': {'object_name': 'FeedFetchHistory'}, + 'exception': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'feed_fetch_history'", 'to': "orm['rss_feeds.Feed']"}), + 'fetch_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'status_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedpage': { + 'Meta': {'object_name': 'FeedPage'}, + 'feed': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'feed_page'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'page_data': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedupdatehistory': { + 'Meta': {'object_name': 'FeedUpdateHistory'}, + 'average_per_feed': ('django.db.models.fields.DecimalField', [], {'max_digits': '4', 'decimal_places': '1'}), + 'fetch_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'number_of_feeds': ('django.db.models.fields.IntegerField', [], {}), + 'seconds_taken': ('django.db.models.fields.IntegerField', [], {}) + }, + 'rss_feeds.feedxml': { + 'Meta': {'object_name': 'FeedXML'}, + 'feed': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'feed_xml'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'rss_xml': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.pagefetchhistory': { + 'Meta': {'object_name': 'PageFetchHistory'}, + 'exception': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'page_fetch_history'", 'to': "orm['rss_feeds.Feed']"}), + 'fetch_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'status_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.storiespermonth': { + 'Meta': {'object_name': 'StoriesPerMonth'}, + 'beginning_of_month': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'stories_per_month'", 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'month': ('django.db.models.fields.IntegerField', [], {}), + 'story_count': ('django.db.models.fields.IntegerField', [], {}), + 'year': ('django.db.models.fields.IntegerField', [], {}) + }, + 'rss_feeds.story': { + 'Meta': {'unique_together': "(('story_feed', 'story_guid_hash'),)", 'object_name': 'Story', 'db_table': "'stories'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'story_author': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.StoryAuthor']"}), + 'story_author_name': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'story_content': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}), + 'story_content_type': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'story_date': ('django.db.models.fields.DateTimeField', [], {}), + 'story_feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'stories'", 'to': "orm['rss_feeds.Feed']"}), + 'story_guid': ('django.db.models.fields.CharField', [], {'max_length': '1000'}), + 'story_guid_hash': ('django.db.models.fields.CharField', [], {'max_length': '40'}), + 'story_original_content': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}), + 'story_past_trim_date': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'story_permalink': ('django.db.models.fields.CharField', [], {'max_length': '1000'}), + 'story_tags': ('django.db.models.fields.CharField', [], {'max_length': '2000', 'null': 'True', 'blank': 'True'}), + 'story_title': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['rss_feeds.Tag']", 'symmetrical': 'False'}) + }, + 'rss_feeds.storyauthor': { + 'Meta': {'object_name': 'StoryAuthor'}, + 'author_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'rss_feeds.tag': { + 'Meta': {'object_name': 'Tag'}, + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + } + } + + complete_apps = ['rss_feeds'] diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 2985d9b0c..945846f1b 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -30,7 +30,8 @@ class Feed(models.Model): feed_tagline = models.CharField(max_length=1024, default="", blank=True, null=True) active = models.BooleanField(default=True) num_subscribers = models.IntegerField(default=0) - last_update = models.DateTimeField(auto_now=True, default=0) + last_update = models.DateTimeField(auto_now=True) + fetched_once = models.BooleanField(default=False) min_to_decay = models.IntegerField(default=15) days_to_trim = models.IntegerField(default=90) creation = models.DateField(auto_now_add=True) @@ -657,8 +658,45 @@ class StoriesPerMonth(models.Model): return month_counts, average_per_month @classmethod - def recount_feed(cls, feed): + def recount_feed(cls, feed, current_counts=None): d = defaultdict(int) - stories = Story.objects.filter(story_feed=feed).extra(select={'year': "EXTRACT(year FROM story_date)", 'month': "EXTRACT(month from story_date)"}).values('year', 'month') + now = datetime.datetime.now() + min_year = now.year + if not current_counts: + current_counts = [] + + # Count stories, aggregate by year and month + stories = Story.objects.filter(story_feed=feed).extra(select={ + 'year': "EXTRACT(year FROM story_date)", + 'month': "EXTRACT(month from story_date)" + }).values('year', 'month') for story in stories: - pass \ No newline at end of file + year = int(story['year']) + d['%s-%s' % (year, int(story['month']))] += 1 + if year < min_year: + min_year = year + + # Add on to existing months, always amending up, never down. (Current month + # is guaranteed to be accurate, since trim_feeds won't delete it until after + # a month. Hacker News can have 1,000+ and still be counted.) + for current_month, current_count in current_counts: + if current_month not in d or d[current_month] < current_count: + d[current_month] = current_count + year = re.findall(r"(\d{4})-\d{1,2}", current_month)[0] + if year < min_year: + min_year = year + + # Assemble a list with 0's filled in for missing months, + # trimming left and right 0's. + months = [] + start = False + for year in range(min_year, now.year+1): + for month in range(1, 12+1): + if datetime.datetime(year, month, 1) < now: + key = '%s-%s' % (year, month) + if d.get(key) or start: + start = True + months.append((key, d.get(key, 0))) + from pprint import pprint + pprint(months) + \ No newline at end of file diff --git a/media/css/reader.css b/media/css/reader.css index bad2e2325..1d18012eb 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -303,6 +303,7 @@ a img { .left-pane { display: none; overflow: hidden !important; + background-color: #D7DDE6; } .left-center { @@ -321,13 +322,30 @@ a img { /* = Feed List = */ /* ============= */ +#NB-feeds-list-loader { + background: transparent url("../img/reader/big_spinner.gif") no-repeat 0 0; + color: #C7CDD6; + font-size: 16px; + height: 51px; + left: 5%; + padding: 5px 0 0 62px; + position: absolute; + text-shadow: 0 1px 0 #E7EDF6; + text-transform: uppercase; + top: 40%; + width: 125px; + z-index: 10; + cursor: default; + display: none; +} + #feed_list ::-moz-selection { -background: transparent; -} + background: transparent; +} #feed_list ::selection { -background: transparent; + background: transparent; } #feed_list { diff --git a/media/img/reader/big_spinner.gif b/media/img/reader/big_spinner.gif new file mode 100644 index 0000000000000000000000000000000000000000..927bab412aa8d07ec5709c36a8093f077947c649 GIT binary patch literal 6494 zcmajkc|6qn_XqIlGoMAD8H_a|h9rCFitI}+iX!_`Su^$-TWDyEHDry*RJQEK2$Ag? zTlT$0QM92p*i99qig|~@ zJXlz68JOxEn%#OIo>`cgdp>c0v2S8d;H1;KE)eWQw~;zFRR7M`Q!X{J5E0T&r^!WzUYTzRu+JM3~F z!XwE0xvp+jtEah6*g#WM-zH=FZ3aQhTF4+$dQV6+N^`})2!NbFmXMf+rtCTGo0*7# zGyu>h-hvC-gWR}Uu97cp2*Du2s+2D#)YYfirB^j^X|O=}mR2>6+2#x8g&Zp!8lM+iqh^5uP)|M;ntINJJ?A7MRKoAQ>6%UpwH$W9#L zH~hn+KUL_U@BCl`QhGOo`2D^wHo$+kafB*-2J{u^ECOA{PSO%zovoBaxEthMg0?*_XV-92J#HYx(k2M962G-QOr^^l?nRPz}f%euUa-YfBsM zII&-S-+m|}5^(p}C$~>r{S3_>LybP97Xd4Q;wJ97sDy);_EGyH3@qm1#4B0^Gr*e% z%$qzBcZh*dJVN-qt2oKr%i4C7ZdDu zlWp_!kA4xxHgNQ~b0j(>=&MlCHc?>Bu*|XII}uJ-jizw?^x`3nNA7@lcd5~F@08O7 zeOIs2`$`_li-WWijJX_bcMwZdKP`{F4mdrqP)=D4CM$v#=VaxjA%8UbBu8-;>l@9I%F%D>CGE0dQ}cg4+^d1V&23-4RCW+=z#_lg4^xReaA1+ z2L=u>1llV*yMp;Th`sRGg~6e@M;$Y}CU_SgJ)1wwSMkb}J}~f@u=aj!W7mfdAAkA$ z>u83VEpyNTA7i zx80t0n5G}bUN?^2=wX98kU@7fn@fupOIpM0p4@T2>JewDgaCHv;us_TjHN@9C~UT} zI0_au%-FLh!zV1H^kdZ^OJ&gNysL1K8GJ!STun|i5DT za^RYY9(P_o9%|%2R^<>*R_6DrxY8t$P!)TpfY8-Az^l^hMI0Kw)rIb@N);U11y+UF z9Hj_P)g!<>InkCV_i|qK!AghEGWp5IE~B;iSDU+5tC*kP+{69nHu)=7pwhc>UmFJA zF0=B_z+E*S{OuS;h*80RWXI`}ouR2&%q)q_VanytK9x;5t`$U4nlc7Ol$GNS#8#2z zk9!Y`kLi!@q#@QP1B< z7A_ErOR$oPkqdTAj}x--^-i!+;x0JK5I%%Gnd5s@Fhd^1R%=9M@>UQf8%>y&eAy~M zW6O7(P@}KIfl~uKVDFI8iE)0Hsp@;e!J3brj6WQD_WW6?<h%EefQ)0 z51Su9{q*zp0Wv_oR*!72{>N?AdyQ^YAAuUgf4k%KvnnAa%S?HxY3W_((wvf#u-`An zQK+gfxLpaEfd?Lp+E?CBOD$So?0Lt3f42dVy3-~8*2gNBJ%g!D!}^}r%NK8pDB(7B zzKw7r%Hln|Jj6gTyMvMP5a4YZ-5m|>_8RAeN~fnk;b2D=x??kkf>&M0QHPNT&wbWbhrN8>Ht(aO%*nWdhs?Jz@@|>;=IJcbPpf-DXM0B3{BQm!+vfA`|r0LzV8cXtDVs!l27m1zQCEom9 zg0UpKhm=`(SvCPKhFMZp zc2pjhVtxdJ&pEgFChv`cg2KGYDxk)#oCmM7I-s-bOIyu~KABEw&SqQqC)MQ;;vb=A&?a{aMe-uR`$*^d{@47z3fH zh)_&|mhC!;DtN_Oe2EA;bk3c1E3}~&@+`*c1Bp|^y+2<%oBr~BDb+%OchNK$2#%&{ zI0JEs0v@MQGX$N_=eT5|VBBQ{NY4{WN!7o=4PpWExr^o85EdJ$SO}B@UR($kvAGzR z;LHg#NmiPT_4vlpM5y(SctA~Re)-`>appB$xUm^G?=eMdL}Ximy-e1=KFmDY+IIbQ zxi}m9?#BwWagMeub@P4XqXCRa7reox%BmQ*RQ5v0zDfIY2m8ePr6+LPLEdc#Y5%>* zHr`bw`&wj4?og}0UoA%TM1q^brs7L2fUzLsidaczee5N8EUFFajyG0jpAZ=A(kF33?p&sK|BaHLj5oTH8JxQg-L6 zQJScv`aG9rY{k~%CG#~b@!J{J{?6bklkO=jSZ-JAbGzjAhXJ0B(IyX}L$vQ(_N)&8 zs;HsERK)xT+DJm?P9lMEcR#03vwM}J(F05jap+}e>Bt*jkJpEsq8pY``ytoNXJT9c z5=GtXejyeyij%s4*dKD|0pcaCbZI~k=DbP-5GxauqX^%)h|-)G`{4YLt$xTW-A4lt zvFK1NzD6pgvVjd^I+|4OpfDD|jEr^34xu@otiHR_V9)S;j?7do9rsjsboTP$$InNe zFDXLbvq^I*H~U^GuWzWo>t8W{r1%g?VDH!D;5%vdr4Kl_18n{~fX{VOHPgSHpC!*u z5S?oNbus+Oq-vbkCat~j8wwtQ`OySgxZJl?AGC^78~;=SIfgAJiYp^!Mq$v zST|LnwAzhNxxWzbpG3*$@pil*HblZ21AzcSv+&!0DM~F16DAEK8Hdxe5G+Z8kx`b8 zq;V%zOGe?V(lWT997*e2n52-VAWTfYSQ`ZR`&P$bN-j#Zjg+=0cZUz|o47x#HeBE@ z#lR0-89?XRvK{E${HRPnaJYmt@B5?>7<=|I91ZIc)AgTw{%xHjsy6nGEa_g4DMQjt z@~C#!7YeTJ6o1Qz_TT^Lg?~`2c99jZ{cHRY0rHXG&FCy}e=HQh4|<*))RfB|EqOsq z&SvEFE?V{obGTx?lv#o4i!AmU5f4hJ{dkNA-zP3vs;?pX5C?mnf@A!?;R^@%oPFYy)0+e9r5MT0ntZm}N`%I6YkPy`@otX-a?Up063fy&dG!f9s7uKqf*1o_=yz zqjG=jHtZ)h3B&Y~izYn8+@fo4nspjE;r9 zE0I}bmez(aWS()ZmhY;Av^kQctj7?_>9YzSdQP-XIJ0S`teXJyn@s{&+_99L0vkbUIQ&!bfOo+2|eWg*q+lah4<0?FmW?%*i zj~AA)hGiKUCY6-O{jF0dmT!10h?~ND4yHue#uD@l!hDY%)zjT0Dk5I(pVi#SDglX&|n&suUpqW?D zS)zVvQq@|7tr%5^b+&a;0*OE7)+uW$0>wGpI_vj?DSQ6p0rjo55(i#s~)YM66FZ52=4Lt0S z)OMeLxPc6zze>2(h-g4xkqJ+$b_TC|Yu|r!kDEfX4d;waXoW5)&oHh8=zN?vo`$8PWNgTV7%0UL6K02rqQ~O%l42M>u0G)DG>pCf4bkK?DXu3f3r-bS zaY2Bwz~|BbQPh12)sf3 zVIzxZ1qeBt>6|X3XB6b|irhFTc)5(8zoraG2f6mB-0P<$%B z*qQ>_vg8VztT_E+RN+{At{@L;+9kUa)<8fGH6!^7S6s@KfLt&>+3(tQ`y1N+)IwRK z>=dX-(WDZrKIEtCm+hk`wxsIF5R$FVil#z5eaz z>#YZYr5lZ?LMT(md$jH8T?*`XQUElqdoFU*7y4K~LkZcZYt9`pG&hm%oeTzi;|gu8 zXkK-ge19BKxyE!&pi=og)m8t*xm6}ti$+7Fj(Vl3JjqSde(P@gVx*EK0X}OEAPKS# zsd(j!IadzqsB(i~oatE#F%YS$PQDyfQ(sqp%c!K0rz*RIaa$5hu4-UkFw-4S#4?w7{&RmdfE|j38cS%*70Y;ubQh?13EQ`tk zuSu(`5N_tc)oS4m_1=L9tCI*Lef|AKY)tRKz~In?U{LEMzO1i)X0r5d$Luim@tH~R zAtyBF4ZrA~2s-z}!V?VR=$;p^biQ9FZ_NJuV+(O<#D3zMwjhMifBJabO_a
+
Everything is on its way...
    diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index 3270c1000..732eaf007 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -307,6 +307,7 @@ class Dispatcher: comment = u'' feed.last_load_time = max(1, delta.seconds) + feed.fetched_once = True try: feed.save() except IntegrityError: