diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index dc9dc8b3d..6db4bb900 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -921,7 +921,7 @@ class Feed(models.Model): def add_update_stories(self, stories, existing_stories, verbose=False): ret_values = dict(new=0, updated=0, same=0, error=0) error_count = self.error_count - new_story_guids = [s.get('guid') for s in stories] + new_story_hashes = [s.get('story_hash') for s in stories] if settings.DEBUG or verbose: logging.debug(" ---> [%-30s] ~FBChecking ~SB%s~SN new/updated against ~SB%s~SN stories" % ( @@ -929,9 +929,9 @@ class Feed(models.Model): len(stories), len(existing_stories.keys()))) @timelimit(2) - def _1(story, story_content, existing_stories, new_story_guids): + def _1(story, story_content, existing_stories, new_story_hashes): existing_story, story_has_changed = self._exists_story(story, story_content, - existing_stories, new_story_guids) + existing_stories, new_story_hashes) return existing_story, story_has_changed for story in stories: @@ -949,7 +949,7 @@ class Feed(models.Model): try: existing_story, story_has_changed = _1(story, story_content, - existing_stories, new_story_guids) + existing_stories, new_story_hashes) except TimeoutError, e: logging.debug(' ---> [%-30s] ~SB~FRExisting story check timed out...' % (unicode(self)[:30])) existing_story = None @@ -988,9 +988,9 @@ class Feed(models.Model): existing_story, _ = MStory.find_story(existing_story.story_feed_id, existing_story.id, original_only=True) - elif existing_story and existing_story.story_guid: + elif existing_story and existing_story.story_hash: existing_story, _ = MStory.find_story(existing_story.story_feed_id, - existing_story.story_guid, + existing_story.story_hash, original_only=True) else: raise MStory.DoesNotExist @@ -1012,7 +1012,7 @@ class Feed(models.Model): # logging.debug("\t\tDiff content: %s" % diff.getDiff()) # if existing_story.story_title != story.get('title'): # logging.debug('\tExisting title / New: : \n\t\t- %s\n\t\t- %s' % (existing_story.story_title, story.get('title'))) - if existing_story.story_guid != story.get('guid'): + if existing_story.story_hash != story.get('story_hash'): self.update_story_with_new_guid(existing_story, story.get('guid')) if settings.DEBUG and False: @@ -1308,11 +1308,11 @@ class Feed(models.Model): link = entry.get('id') return link - def _exists_story(self, story, story_content, existing_stories, new_story_guids): + def _exists_story(self, story, story_content, existing_stories, new_story_hashes): story_in_system = None story_has_changed = False story_link = self.get_permalink(story) - existing_stories_guids = existing_stories.keys() + existing_stories_hashes = existing_stories.keys() story_pub_date = story.get('published') # story_published_now = story.get('published_now', False) # start_date = story_pub_date - datetime.timedelta(hours=8) @@ -1322,7 +1322,22 @@ class Feed(models.Model): content_ratio = 0 # existing_story_pub_date = existing_story.story_date # print 'Story pub date: %s %s' % (story_published_now, story_pub_date) + + if isinstance(existing_story.id, unicode): + # Correcting a MongoDB bug + existing_story.story_guid = existing_story.id + if story.get('story_hash') == existing_story.story_hash: + story_in_system = existing_story + elif (story.get('story_hash') in existing_stories_hashes and + story.get('story_hash') != existing_story.story_hash): + # Story already exists but is not this one + continue + elif (existing_story.story_hash in new_story_hashes and + story.get('story_hash') != existing_story.story_hash): + # Story coming up later + continue + if 'story_latest_content_z' in existing_story: existing_story_content = unicode(zlib.decompress(existing_story.story_latest_content_z)) elif 'story_latest_content' in existing_story: @@ -1334,17 +1349,6 @@ class Feed(models.Model): else: existing_story_content = u'' - if isinstance(existing_story.id, unicode): - existing_story.story_guid = existing_story.id - if (story.get('guid') in existing_stories_guids and - story.get('guid') != existing_story.story_guid): - continue - elif story.get('guid') == existing_story.story_guid: - story_in_system = existing_story - elif (existing_story.story_guid in new_story_guids and - story.get('guid') != existing_story.story_guid): - # Story coming up later - continue # Title distance + content distance, checking if story changed story_title_difference = abs(levenshtein_distance(story.get('title'), @@ -1692,10 +1696,18 @@ class MStory(mongo.Document): def guid_hash(self): return hashlib.sha1(self.story_guid).hexdigest()[:6] + @classmethod + def guid_hash_unsaved(self, guid): + return hashlib.sha1(guid).hexdigest()[:6] + @property def feed_guid_hash(self): return "%s:%s" % (self.story_feed_id, self.guid_hash) - + + @classmethod + def feed_guid_hash_unsaved(cls, feed_id, guid): + return "%s:%s" % (feed_id, cls.guid_hash_unsaved(guid)) + @property def decoded_story_title(self): h = HTMLParser.HTMLParser() diff --git a/clients/ios/Classes/FeedDetailViewController.m b/clients/ios/Classes/FeedDetailViewController.m index f681d04f8..6a92ea7bd 100644 --- a/clients/ios/Classes/FeedDetailViewController.m +++ b/clients/ios/Classes/FeedDetailViewController.m @@ -1269,7 +1269,8 @@ FeedDetailTableCell *cell = (FeedDetailTableCell*) [tableView cellForRowAtIndexPath:indexPath]; NSInteger storyIndex = [storiesCollection indexFromLocation:indexPath.row]; NSDictionary *story = [[storiesCollection activeFeedStories] objectAtIndex:storyIndex]; - if (appDelegate.activeStory && + if (UI_USER_INTERFACE_IDIOM() == UIUserInterfaceIdiomPad && + appDelegate.activeStory && [[story objectForKey:@"story_hash"] isEqualToString:[appDelegate.activeStory objectForKey:@"story_hash"]]) { return; diff --git a/clients/ios/Classes/FolderTitleView.m b/clients/ios/Classes/FolderTitleView.m index 3873b49ab..ab443bec7 100644 --- a/clients/ios/Classes/FolderTitleView.m +++ b/clients/ios/Classes/FolderTitleView.m @@ -145,10 +145,24 @@ invisibleHeaderButton.frame = CGRectMake(0, 0, customView.frame.size.width, customView.frame.size.height); invisibleHeaderButton.alpha = .1; invisibleHeaderButton.tag = section; - [invisibleHeaderButton addTarget:appDelegate.feedsViewController action:@selector(didSelectSectionHeader:) forControlEvents:UIControlEventTouchUpInside]; - [invisibleHeaderButton addTarget:appDelegate.feedsViewController action:@selector(sectionTapped:) forControlEvents:UIControlEventTouchDown]; - [invisibleHeaderButton addTarget:appDelegate.feedsViewController action:@selector(sectionUntapped:) forControlEvents:UIControlEventTouchUpInside]; - [invisibleHeaderButton addTarget:appDelegate.feedsViewController action:@selector(sectionUntappedOutside:) forControlEvents:UIControlEventTouchUpOutside]; + [invisibleHeaderButton addTarget:appDelegate.feedsViewController + action:@selector(didSelectSectionHeader:) + forControlEvents:UIControlEventTouchUpInside]; + [invisibleHeaderButton addTarget:appDelegate.feedsViewController + action:@selector(sectionTapped:) + forControlEvents:UIControlEventTouchDown]; + [invisibleHeaderButton addTarget:appDelegate.feedsViewController + action:@selector(sectionUntapped:) + forControlEvents:UIControlEventTouchUpInside]; + [invisibleHeaderButton addTarget:appDelegate.feedsViewController + action:@selector(sectionUntappedOutside:) + forControlEvents:UIControlEventTouchUpOutside]; + [invisibleHeaderButton addTarget:appDelegate.feedsViewController + action:@selector(sectionUntappedOutside:) + forControlEvents:UIControlEventTouchCancel]; + [invisibleHeaderButton addTarget:appDelegate.feedsViewController + action:@selector(sectionUntappedOutside:) + forControlEvents:UIControlEventTouchDragOutside]; [customView addSubview:invisibleHeaderButton]; if (!appDelegate.hasNoSites) { diff --git a/clients/ios/Classes/StoriesCollection.m b/clients/ios/Classes/StoriesCollection.m index 87e540a6e..5d84e9e26 100644 --- a/clients/ios/Classes/StoriesCollection.m +++ b/clients/ios/Classes/StoriesCollection.m @@ -40,6 +40,7 @@ if (self = [super init]) { self.visibleUnreadCount = 0; self.appDelegate = (NewsBlurAppDelegate *)[[UIApplication sharedApplication] delegate]; + self.activeClassifiers = [NSMutableDictionary dictionary]; } return self; diff --git a/clients/ios/Classes/StoryPageControl.m b/clients/ios/Classes/StoryPageControl.m index 3817e536c..2dbc1f843 100644 --- a/clients/ios/Classes/StoryPageControl.m +++ b/clients/ios/Classes/StoryPageControl.m @@ -788,7 +788,8 @@ buttonNext.enabled = YES; NSInteger nextIndex = [appDelegate.storiesCollection indexOfNextUnreadStory]; NSInteger unreadCount = [appDelegate unreadCount]; - if ((nextIndex == -1 && unreadCount > 0) || + BOOL pageFinished = self.appDelegate.feedDetailViewController.pageFinished; + if ((nextIndex == -1 && unreadCount > 0 && !pageFinished) || nextIndex != -1) { [buttonNext setTitle:[@"Next" uppercaseString] forState:UIControlStateNormal]; [buttonNext setBackgroundImage:[UIImage imageNamed:@"traverse_next.png"] @@ -986,12 +987,14 @@ FeedDetailViewController *fdvc = self.appDelegate.feedDetailViewController; NSInteger nextLocation = [appDelegate.storiesCollection locationOfNextUnreadStory]; NSInteger unreadCount = [appDelegate unreadCount]; + BOOL pageFinished = self.appDelegate.feedDetailViewController.pageFinished; + [self.loadingIndicator stopAnimating]; [self endTouchDown:sender]; // NSLog(@"doNextUnreadStory: %d (out of %d)", nextLocation, unreadCount); - if (nextLocation == -1 && unreadCount > 0 && + if (nextLocation == -1 && unreadCount > 0 && !pageFinished && appDelegate.storiesCollection.feedPage < 100) { [self.loadingIndicator startAnimating]; self.circularProgressView.hidden = YES; diff --git a/clients/ios/Classes/TrainerViewController.m b/clients/ios/Classes/TrainerViewController.m index 4805d5c6c..5307842f7 100644 --- a/clients/ios/Classes/TrainerViewController.m +++ b/clients/ios/Classes/TrainerViewController.m @@ -245,27 +245,47 @@ } - (NSString *)makeFeedAuthors { + NSString *feedId = [self feedId]; NSString *feedAuthors = @""; NSArray *authorArray = appDelegate.storiesCollection.activePopularAuthors; - NSString *feedId = [self feedId]; + NSMutableArray *userAuthorArray = [NSMutableArray array]; + for (NSString *trainedAuthor in [[[appDelegate.storiesCollection.activeClassifiers objectForKey:feedId] + objectForKey:@"authors"] allKeys]) { + BOOL found = NO; + for (NSArray *classifierAuthor in authorArray) { + if ([trainedAuthor isEqualToString:[classifierAuthor objectAtIndex:0]]) { + found = YES; + break; + } + } + if (!found) { + [userAuthorArray addObject:@[trainedAuthor, [NSNumber numberWithInt:0]]]; + } + } + NSArray *authors = [userAuthorArray arrayByAddingObjectsFromArray:authorArray]; - if ([authorArray count] > 0) { + if ([authors count] > 0) { NSMutableArray *authorStrings = [NSMutableArray array]; - for (NSArray *authorObj in authorArray) { + for (NSArray *authorObj in authors) { NSString *author = [authorObj objectAtIndex:0]; int authorCount = [[authorObj objectAtIndex:1] intValue]; int authorScore = [[[[appDelegate.storiesCollection.activeClassifiers objectForKey:feedId] objectForKey:@"authors"] objectForKey:author] intValue]; + NSString *authorCountString = @""; + if (authorCount) { + authorCountString = [NSString stringWithFormat:@"×  %d", + authorCount]; + } NSString *authorHtml = [NSString stringWithFormat:@"
" " %@" - " ×  %d" + " %@" "
", author, authorScore > 0 ? @"NB-story-author-positive" : authorScore < 0 ? @"NB-story-author-negative" : @"", [self makeClassifier:author withType:@"author" score:authorScore], - authorCount]; + authorCountString]; [authorStrings addObject:authorHtml]; } feedAuthors = [NSString @@ -325,24 +345,44 @@ NSString *feedId = [self feedId]; NSString *feedTags = @""; NSArray *tagArray = appDelegate.storiesCollection.activePopularTags; - - if ([tagArray count] > 0) { + NSMutableArray *userTagArray = [NSMutableArray array]; + for (NSString *trainedTag in [[[appDelegate.storiesCollection.activeClassifiers objectForKey:feedId] + objectForKey:@"tags"] allKeys]) { + BOOL found = NO; + for (NSArray *classifierTag in tagArray) { + if ([trainedTag isEqualToString:[classifierTag objectAtIndex:0]]) { + found = YES; + break; + } + } + if (!found) { + [userTagArray addObject:@[trainedTag, [NSNumber numberWithInt:0]]]; + } + } + NSArray *tags = [userTagArray arrayByAddingObjectsFromArray:tagArray]; + + if ([tags count] > 0) { NSMutableArray *tagStrings = [NSMutableArray array]; - for (NSArray *tagObj in tagArray) { + for (NSArray *tagObj in tags) { NSString *tag = [tagObj objectAtIndex:0]; int tagCount = [[tagObj objectAtIndex:1] intValue]; int tagScore = [[[[appDelegate.storiesCollection.activeClassifiers objectForKey:feedId] objectForKey:@"tags"] objectForKey:tag] intValue]; + NSString *tagCountString = @""; + if (tagCount) { + tagCountString = [NSString stringWithFormat:@"×  %d", + tagCount]; + } NSString *tagHtml = [NSString stringWithFormat:@"
" " %@" - " ×  %d" + " %@" "
", tag, tagScore > 0 ? @"NB-story-tag-positive" : tagScore < 0 ? @"NB-story-tag-negative" : @"", [self makeClassifier:tag withType:@"Tag" score:tagScore], - tagCount]; + tagCountString]; [tagStrings addObject:tagHtml]; } feedTags = [NSString @@ -485,14 +525,7 @@ shouldStartLoadWithRequest:(NSURLRequest *)request NSURL *url = [request URL]; NSArray *urlComponents = [url pathComponents]; NSString *action = @""; - NSString *feedId; - if (appDelegate.storiesCollection.isSocialView || appDelegate.storiesCollection.isSocialRiverView) { - feedId = [NSString stringWithFormat:@"%@", [appDelegate.activeStory - objectForKey:@"story_feed_id"]]; - } else { - feedId = [NSString stringWithFormat:@"%@", [appDelegate.storiesCollection.activeFeed - objectForKey:@"id"]]; - } + NSString *feedId = [self feedId]; if ([urlComponents count] > 1) { action = [NSString stringWithFormat:@"%@", [urlComponents objectAtIndex:1]]; diff --git a/clients/ios/NewsBlur-iPhone-Info.plist b/clients/ios/NewsBlur-iPhone-Info.plist index f3d210b7b..79f3b80a5 100644 --- a/clients/ios/NewsBlur-iPhone-Info.plist +++ b/clients/ios/NewsBlur-iPhone-Info.plist @@ -19,7 +19,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 4.0.2 + 4.0.3 CFBundleSignature ???? CFBundleURLTypes @@ -58,7 +58,7 @@ CFBundleVersion - 4.0.2 + 4.0.3 FacebookAppID 230426707030569 LSRequiresIPhoneOS diff --git a/clients/ios/NewsBlur.xcodeproj/project.pbxproj b/clients/ios/NewsBlur.xcodeproj/project.pbxproj index 5daa871ad..49e6f9c08 100755 --- a/clients/ios/NewsBlur.xcodeproj/project.pbxproj +++ b/clients/ios/NewsBlur.xcodeproj/project.pbxproj @@ -3488,7 +3488,7 @@ "-all_load", ); PRODUCT_NAME = NewsBlur; - PROVISIONING_PROFILE = ""; + PROVISIONING_PROFILE = "4753388F-4504-40B3-AB43-61FB2E9FDEF8"; TARGETED_DEVICE_FAMILY = "1,2"; "WARNING_CFLAGS[arch=*]" = "-Wall"; }; @@ -3525,7 +3525,7 @@ "-all_load", ); PRODUCT_NAME = NewsBlur; - PROVISIONING_PROFILE = ""; + PROVISIONING_PROFILE = "4753388F-4504-40B3-AB43-61FB2E9FDEF8"; TARGETED_DEVICE_FAMILY = "1,2"; VALIDATE_PRODUCT = YES; }; diff --git a/fabfile.py b/fabfile.py index 29eeff336..0c086d942 100644 --- a/fabfile.py +++ b/fabfile.py @@ -88,7 +88,7 @@ def list_do(): total_cost = 0 for droplet in droplets: roledef = re.split(r"([0-9]+)", droplet.name)[0] - cost = int(sizes[droplet.size_id]) * 10 + cost = int(sizes.get(droplet.size_id, 96)) * 10 role_costs[roledef] += cost total_cost += cost @@ -930,7 +930,9 @@ def setup_munin(): sudo('/etc/init.d/spawn_fcgi_munin_html stop') sudo('/etc/init.d/spawn_fcgi_munin_html start') sudo('update-rc.d spawn_fcgi_munin_html defaults') - sudo('/etc/init.d/munin-node restart') + sudo('/etc/init.d/munin-node stop') + time.sleep(2) + sudo('/etc/init.d/munin-node start') with settings(warn_only=True): sudo('chown nginx.www-data /var/log/munin/munin-cgi*') sudo('chown nginx.www-data /usr/lib/cgi-bin/munin-cgi*') @@ -949,7 +951,10 @@ def setup_db_munin(): run('git clone git://github.com/samuel/python-munin.git') with cd(os.path.join(env.VENDOR_PATH, 'python-munin')): run('sudo python setup.py install') - sudo('/etc/init.d/munin-node restart') + sudo('/etc/init.d/munin-node stop') + time.sleep(2) + sudo('/etc/init.d/munin-node start') + def enable_celerybeat(): with cd(env.NEWSBLUR_PATH): @@ -1043,7 +1048,7 @@ def setup_do(name, size=2, image=None): ssh_key_ids = [str(k.id) for k in doapi.all_ssh_keys()] region_id = doapi.regions()[0].id if not image: - IMAGE_NAME = "Ubuntu 13.04 x64" + IMAGE_NAME = "Ubuntu 13.10 x64" images = dict((s.name, s.id) for s in doapi.images()) image_id = images[IMAGE_NAME] else: diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index 8a175e2a5..e39c0c00d 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -226,20 +226,21 @@ class ProcessFeed: # Compare new stories to existing stories, adding and updating start_date = datetime.datetime.utcnow() - story_guids = [] + story_hashes = [] stories = [] for entry in self.fpf.entries: story = pre_process_story(entry) if story.get('published') < start_date: start_date = story.get('published') + story['story_hash'] = MStory.feed_guid_hash_unsaved(self.feed.pk, story.get('guid')) stories.append(story) - story_guids.append(story.get('guid')) + story_hashes.append(story.get('story_hash')) - existing_stories = dict((s.story_guid, s) for s in MStory.objects( - # story_guid__in=story_guids, - story_date__gte=start_date, - story_feed_id=self.feed.pk - ).limit(max(int(len(story_guids)*1.5), 10))) + existing_stories = dict((s.story_hash, s) for s in MStory.objects( + story_hash__in=story_hashes, + # story_date__gte=start_date, + # story_feed_id=self.feed.pk + )) ret_values = self.feed.add_update_stories(stories, existing_stories, verbose=self.options['verbose'])