public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-09-18 21:50:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into search

Browse source 
* master:
  iOS app versin 4.0.3.
  Fixing intelligence trainer when training in folder.
  Showing user-created training tags/authors in feed trainer.
  Correctly handling drag events on folder titles.
  Showing correct Done/Next button status based on actual stories loaded. This one's for @immad.
  Moving to stricter story diff check. Using story hashes instead of grab bag by date when checking existing stories.
  Fixing digitalocean's missing 96GB size.
This commit is contained in:
Samuel Clay 2014-04-18 16:40:52 -07:00
commit 7d0ba4e829
10 changed files with 131 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
apps/rss_feeds/models.py
View file

@ -921,7 +921,7 @@ class Feed(models.Model):
def add_update_stories(self, stories, existing_stories, verbose=False):
ret_values = dict(new=0, updated=0, same=0, error=0)
error_count = self.error_count
new_story_guids = [s.get('guid') for s in stories]
new_story_hashes = [s.get('story_hash') for s in stories]
if settings.DEBUG or verbose:
logging.debug(" ---> [%-30s] ~FBChecking ~SB%s~SN new/updated against ~SB%s~SN stories" % (
@ -929,9 +929,9 @@ class Feed(models.Model):
len(stories),
len(existing_stories.keys())))
@timelimit(2)
def _1(story, story_content, existing_stories, new_story_guids):
def _1(story, story_content, existing_stories, new_story_hashes):
existing_story, story_has_changed = self._exists_story(story, story_content,
existing_stories, new_story_guids)
existing_stories, new_story_hashes)
return existing_story, story_has_changed
for story in stories:
@ -949,7 +949,7 @@ class Feed(models.Model):
try:
existing_story, story_has_changed = _1(story, story_content,
existing_stories, new_story_guids)
existing_stories, new_story_hashes)
except TimeoutError, e:
logging.debug(' ---> [%-30s] ~SB~FRExisting story check timed out...' % (unicode(self)[:30]))
existing_story = None
@ -988,9 +988,9 @@ class Feed(models.Model):
existing_story, _ = MStory.find_story(existing_story.story_feed_id,
existing_story.id,
original_only=True)
elif existing_story and existing_story.story_guid:
elif existing_story and existing_story.story_hash:
existing_story, _ = MStory.find_story(existing_story.story_feed_id,
existing_story.story_guid,
existing_story.story_hash,
original_only=True)
else:
raise MStory.DoesNotExist
@ -1012,7 +1012,7 @@ class Feed(models.Model):
# logging.debug("\t\tDiff content: %s" % diff.getDiff())
# if existing_story.story_title != story.get('title'):
# logging.debug('\tExisting title / New: : \n\t\t- %s\n\t\t- %s' % (existing_story.story_title, story.get('title')))
if existing_story.story_guid != story.get('guid'):
if existing_story.story_hash != story.get('story_hash'):
self.update_story_with_new_guid(existing_story, story.get('guid'))
if settings.DEBUG and False:
@ -1308,11 +1308,11 @@ class Feed(models.Model):
link = entry.get('id')
return link
def _exists_story(self, story, story_content, existing_stories, new_story_guids):
def _exists_story(self, story, story_content, existing_stories, new_story_hashes):
story_in_system = None
story_has_changed = False
story_link = self.get_permalink(story)
existing_stories_guids = existing_stories.keys()
existing_stories_hashes = existing_stories.keys()
story_pub_date = story.get('published')
# story_published_now = story.get('published_now', False)
# start_date = story_pub_date - datetime.timedelta(hours=8)
@ -1322,7 +1322,22 @@ class Feed(models.Model):
content_ratio = 0
# existing_story_pub_date = existing_story.story_date
# print 'Story pub date: %s %s' % (story_published_now, story_pub_date)
if isinstance(existing_story.id, unicode):
# Correcting a MongoDB bug
existing_story.story_guid = existing_story.id
if story.get('story_hash') == existing_story.story_hash:
story_in_system = existing_story
elif (story.get('story_hash') in existing_stories_hashes and
story.get('story_hash') != existing_story.story_hash):
# Story already exists but is not this one
continue
elif (existing_story.story_hash in new_story_hashes and
story.get('story_hash') != existing_story.story_hash):
# Story coming up later
continue
if 'story_latest_content_z' in existing_story:
existing_story_content = unicode(zlib.decompress(existing_story.story_latest_content_z))
elif 'story_latest_content' in existing_story:
@ -1334,17 +1349,6 @@ class Feed(models.Model):
else:
existing_story_content = u''
if isinstance(existing_story.id, unicode):
existing_story.story_guid = existing_story.id
if (story.get('guid') in existing_stories_guids and
story.get('guid') != existing_story.story_guid):
continue
elif story.get('guid') == existing_story.story_guid:
story_in_system = existing_story
elif (existing_story.story_guid in new_story_guids and
story.get('guid') != existing_story.story_guid):
# Story coming up later
continue
# Title distance + content distance, checking if story changed
story_title_difference = abs(levenshtein_distance(story.get('title'),
@ -1692,10 +1696,18 @@ class MStory(mongo.Document):
def guid_hash(self):
return hashlib.sha1(self.story_guid).hexdigest()[:6]
@classmethod
def guid_hash_unsaved(self, guid):
return hashlib.sha1(guid).hexdigest()[:6]
@property
def feed_guid_hash(self):
return "%s:%s" % (self.story_feed_id, self.guid_hash)
@classmethod
def feed_guid_hash_unsaved(cls, feed_id, guid):
return "%s:%s" % (feed_id, cls.guid_hash_unsaved(guid))
@property
def decoded_story_title(self):
h = HTMLParser.HTMLParser()

3
clients/ios/Classes/FeedDetailViewController.m
View file

@ -1269,7 +1269,8 @@
FeedDetailTableCell *cell = (FeedDetailTableCell*) [tableView cellForRowAtIndexPath:indexPath];
NSInteger storyIndex = [storiesCollection indexFromLocation:indexPath.row];
NSDictionary *story = [[storiesCollection activeFeedStories] objectAtIndex:storyIndex];
if (appDelegate.activeStory &&
if (UI_USER_INTERFACE_IDIOM() == UIUserInterfaceIdiomPad &&
appDelegate.activeStory &&
[[story objectForKey:@"story_hash"]
isEqualToString:[appDelegate.activeStory objectForKey:@"story_hash"]]) {
return;

22
clients/ios/Classes/FolderTitleView.m
View file

@ -145,10 +145,24 @@
invisibleHeaderButton.frame = CGRectMake(0, 0, customView.frame.size.width, customView.frame.size.height);
invisibleHeaderButton.alpha = .1;
invisibleHeaderButton.tag = section;
[invisibleHeaderButton addTarget:appDelegate.feedsViewController action:@selector(didSelectSectionHeader:) forControlEvents:UIControlEventTouchUpInside];
[invisibleHeaderButton addTarget:appDelegate.feedsViewController action:@selector(sectionTapped:) forControlEvents:UIControlEventTouchDown];
[invisibleHeaderButton addTarget:appDelegate.feedsViewController action:@selector(sectionUntapped:) forControlEvents:UIControlEventTouchUpInside];
[invisibleHeaderButton addTarget:appDelegate.feedsViewController action:@selector(sectionUntappedOutside:) forControlEvents:UIControlEventTouchUpOutside];
[invisibleHeaderButton addTarget:appDelegate.feedsViewController
action:@selector(didSelectSectionHeader:)
forControlEvents:UIControlEventTouchUpInside];
[invisibleHeaderButton addTarget:appDelegate.feedsViewController
action:@selector(sectionTapped:)
forControlEvents:UIControlEventTouchDown];
[invisibleHeaderButton addTarget:appDelegate.feedsViewController
action:@selector(sectionUntapped:)
forControlEvents:UIControlEventTouchUpInside];
[invisibleHeaderButton addTarget:appDelegate.feedsViewController
action:@selector(sectionUntappedOutside:)
forControlEvents:UIControlEventTouchUpOutside];
[invisibleHeaderButton addTarget:appDelegate.feedsViewController
action:@selector(sectionUntappedOutside:)
forControlEvents:UIControlEventTouchCancel];
[invisibleHeaderButton addTarget:appDelegate.feedsViewController
action:@selector(sectionUntappedOutside:)
forControlEvents:UIControlEventTouchDragOutside];
[customView addSubview:invisibleHeaderButton];
if (!appDelegate.hasNoSites) {

1
clients/ios/Classes/StoriesCollection.m
View file

@ -40,6 +40,7 @@
if (self = [super init]) {
self.visibleUnreadCount = 0;
self.appDelegate = (NewsBlurAppDelegate *)[[UIApplication sharedApplication] delegate];
self.activeClassifiers = [NSMutableDictionary dictionary];
}
return self;

7
clients/ios/Classes/StoryPageControl.m
View file

@ -788,7 +788,8 @@
buttonNext.enabled = YES;
NSInteger nextIndex = [appDelegate.storiesCollection indexOfNextUnreadStory];
NSInteger unreadCount = [appDelegate unreadCount];
if ((nextIndex == -1 && unreadCount > 0) ||
BOOL pageFinished = self.appDelegate.feedDetailViewController.pageFinished;
if ((nextIndex == -1 && unreadCount > 0 && !pageFinished) ||
nextIndex != -1) {
[buttonNext setTitle:[@"Next" uppercaseString] forState:UIControlStateNormal];
[buttonNext setBackgroundImage:[UIImage imageNamed:@"traverse_next.png"]
@ -986,12 +987,14 @@
FeedDetailViewController *fdvc = self.appDelegate.feedDetailViewController;
NSInteger nextLocation = [appDelegate.storiesCollection locationOfNextUnreadStory];
NSInteger unreadCount = [appDelegate unreadCount];
BOOL pageFinished = self.appDelegate.feedDetailViewController.pageFinished;
[self.loadingIndicator stopAnimating];
[self endTouchDown:sender];
// NSLog(@"doNextUnreadStory: %d (out of %d)", nextLocation, unreadCount);
if (nextLocation == -1 && unreadCount > 0 &&
if (nextLocation == -1 && unreadCount > 0 && !pageFinished &&
appDelegate.storiesCollection.feedPage < 100) {
[self.loadingIndicator startAnimating];
self.circularProgressView.hidden = YES;

69
clients/ios/Classes/TrainerViewController.m
View file

@ -245,27 +245,47 @@
}
- (NSString *)makeFeedAuthors {
NSString *feedId = [self feedId];
NSString *feedAuthors = @"";
NSArray *authorArray = appDelegate.storiesCollection.activePopularAuthors;
NSString *feedId = [self feedId];
NSMutableArray *userAuthorArray = [NSMutableArray array];
for (NSString *trainedAuthor in [[[appDelegate.storiesCollection.activeClassifiers objectForKey:feedId]
objectForKey:@"authors"] allKeys]) {
BOOL found = NO;
for (NSArray *classifierAuthor in authorArray) {
if ([trainedAuthor isEqualToString:[classifierAuthor objectAtIndex:0]]) {
found = YES;
break;
}
}
if (!found) {
[userAuthorArray addObject:@[trainedAuthor, [NSNumber numberWithInt:0]]];
}
}
NSArray *authors = [userAuthorArray arrayByAddingObjectsFromArray:authorArray];
if ([authorArray count] > 0) {
if ([authors count] > 0) {
NSMutableArray *authorStrings = [NSMutableArray array];
for (NSArray *authorObj in authorArray) {
for (NSArray *authorObj in authors) {
NSString *author = [authorObj objectAtIndex:0];
int authorCount = [[authorObj objectAtIndex:1] intValue];
int authorScore = [[[[appDelegate.storiesCollection.activeClassifiers objectForKey:feedId]
objectForKey:@"authors"]
objectForKey:author] intValue];
NSString *authorCountString = @"";
if (authorCount) {
authorCountString = [NSString stringWithFormat:@"<span class=\"NB-classifier-count\">&times;&nbsp; %d</span>",
authorCount];
}
NSString *authorHtml = [NSString stringWithFormat:@"<div class=\"NB-classifier-container\">"
" <a href=\"http://ios.newsblur.com/classify-author/%@\" "
" class=\"NB-story-author %@\">%@</a>"
" <span class=\"NB-classifier-count\">&times;&nbsp; %d</span>"
" %@"
"</div>",
author,
authorScore > 0 ? @"NB-story-author-positive" : authorScore < 0 ? @"NB-story-author-negative" : @"",
[self makeClassifier:author withType:@"author" score:authorScore],
authorCount];
authorCountString];
[authorStrings addObject:authorHtml];
}
feedAuthors = [NSString
@ -325,24 +345,44 @@
NSString *feedId = [self feedId];
NSString *feedTags = @"";
NSArray *tagArray = appDelegate.storiesCollection.activePopularTags;
if ([tagArray count] > 0) {
NSMutableArray *userTagArray = [NSMutableArray array];
for (NSString *trainedTag in [[[appDelegate.storiesCollection.activeClassifiers objectForKey:feedId]
objectForKey:@"tags"] allKeys]) {
BOOL found = NO;
for (NSArray *classifierTag in tagArray) {
if ([trainedTag isEqualToString:[classifierTag objectAtIndex:0]]) {
found = YES;
break;
}
}
if (!found) {
[userTagArray addObject:@[trainedTag, [NSNumber numberWithInt:0]]];
}
}
NSArray *tags = [userTagArray arrayByAddingObjectsFromArray:tagArray];
if ([tags count] > 0) {
NSMutableArray *tagStrings = [NSMutableArray array];
for (NSArray *tagObj in tagArray) {
for (NSArray *tagObj in tags) {
NSString *tag = [tagObj objectAtIndex:0];
int tagCount = [[tagObj objectAtIndex:1] intValue];
int tagScore = [[[[appDelegate.storiesCollection.activeClassifiers objectForKey:feedId]
objectForKey:@"tags"]
objectForKey:tag] intValue];
NSString *tagCountString = @"";
if (tagCount) {
tagCountString = [NSString stringWithFormat:@"<span class=\"NB-classifier-count\">&times;&nbsp; %d</span>",
tagCount];
}
NSString *tagHtml = [NSString stringWithFormat:@"<div class=\"NB-classifier-container\">"
" <a href=\"http://ios.newsblur.com/classify-tag/%@\" "
" class=\"NB-story-tag %@\">%@</a>"
" <span class=\"NB-classifier-count\">&times;&nbsp; %d</span>"
" %@"
"</div>",
tag,
tagScore > 0 ? @"NB-story-tag-positive" : tagScore < 0 ? @"NB-story-tag-negative" : @"",
[self makeClassifier:tag withType:@"Tag" score:tagScore],
tagCount];
tagCountString];
[tagStrings addObject:tagHtml];
}
feedTags = [NSString
@ -485,14 +525,7 @@ shouldStartLoadWithRequest:(NSURLRequest *)request
NSURL *url = [request URL];
NSArray *urlComponents = [url pathComponents];
NSString *action = @"";
NSString *feedId;
if (appDelegate.storiesCollection.isSocialView || appDelegate.storiesCollection.isSocialRiverView) {
feedId = [NSString stringWithFormat:@"%@", [appDelegate.activeStory
objectForKey:@"story_feed_id"]];
} else {
feedId = [NSString stringWithFormat:@"%@", [appDelegate.storiesCollection.activeFeed
objectForKey:@"id"]];
}
NSString *feedId = [self feedId];
if ([urlComponents count] > 1) {
action = [NSString stringWithFormat:@"%@", [urlComponents objectAtIndex:1]];

4
clients/ios/NewsBlur-iPhone-Info.plist
View file

@ -19,7 +19,7 @@
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>4.0.2</string>
<string>4.0.3</string>
<key>CFBundleSignature</key>
<string>????</string>
<key>CFBundleURLTypes</key>
@ -58,7 +58,7 @@
</dict>
</array>
<key>CFBundleVersion</key>
<string>4.0.2</string>
<string>4.0.3</string>
<key>FacebookAppID</key>
<string>230426707030569</string>
<key>LSRequiresIPhoneOS</key>

4
clients/ios/NewsBlur.xcodeproj/project.pbxproj
View file

@ -3488,7 +3488,7 @@
"-all_load",
);
PRODUCT_NAME = NewsBlur;
PROVISIONING_PROFILE = "";
PROVISIONING_PROFILE = "4753388F-4504-40B3-AB43-61FB2E9FDEF8";
TARGETED_DEVICE_FAMILY = "1,2";
"WARNING_CFLAGS[arch=*]" = "-Wall";
};
@ -3525,7 +3525,7 @@
"-all_load",
);
PRODUCT_NAME = NewsBlur;
PROVISIONING_PROFILE = "";
PROVISIONING_PROFILE = "4753388F-4504-40B3-AB43-61FB2E9FDEF8";
TARGETED_DEVICE_FAMILY = "1,2";
VALIDATE_PRODUCT = YES;
};

13
fabfile.py vendored
View file

@ -88,7 +88,7 @@ def list_do():
total_cost = 0
for droplet in droplets:
roledef = re.split(r"([0-9]+)", droplet.name)[0]
cost = int(sizes[droplet.size_id]) * 10
cost = int(sizes.get(droplet.size_id, 96)) * 10
role_costs[roledef] += cost
total_cost += cost
@ -930,7 +930,9 @@ def setup_munin():
sudo('/etc/init.d/spawn_fcgi_munin_html stop')
sudo('/etc/init.d/spawn_fcgi_munin_html start')
sudo('update-rc.d spawn_fcgi_munin_html defaults')
sudo('/etc/init.d/munin-node restart')
sudo('/etc/init.d/munin-node stop')
time.sleep(2)
sudo('/etc/init.d/munin-node start')
with settings(warn_only=True):
sudo('chown nginx.www-data /var/log/munin/munin-cgi*')
sudo('chown nginx.www-data /usr/lib/cgi-bin/munin-cgi*')
@ -949,7 +951,10 @@ def setup_db_munin():
run('git clone git://github.com/samuel/python-munin.git')
with cd(os.path.join(env.VENDOR_PATH, 'python-munin')):
run('sudo python setup.py install')
sudo('/etc/init.d/munin-node restart')
sudo('/etc/init.d/munin-node stop')
time.sleep(2)
sudo('/etc/init.d/munin-node start')
def enable_celerybeat():
with cd(env.NEWSBLUR_PATH):
@ -1043,7 +1048,7 @@ def setup_do(name, size=2, image=None):
ssh_key_ids = [str(k.id) for k in doapi.all_ssh_keys()]
region_id = doapi.regions()[0].id
if not image:
IMAGE_NAME = "Ubuntu 13.04 x64"
IMAGE_NAME = "Ubuntu 13.10 x64"
images = dict((s.name, s.id) for s in doapi.images())
image_id = images[IMAGE_NAME]
else:

15
utils/feed_fetcher.py
View file

@ -226,20 +226,21 @@ class ProcessFeed:
# Compare new stories to existing stories, adding and updating
start_date = datetime.datetime.utcnow()
story_guids = []
story_hashes = []
stories = []
for entry in self.fpf.entries:
story = pre_process_story(entry)
if story.get('published') < start_date:
start_date = story.get('published')
story['story_hash'] = MStory.feed_guid_hash_unsaved(self.feed.pk, story.get('guid'))
stories.append(story)
story_guids.append(story.get('guid'))
story_hashes.append(story.get('story_hash'))
existing_stories = dict((s.story_guid, s) for s in MStory.objects(
# story_guid__in=story_guids,
story_date__gte=start_date,
story_feed_id=self.feed.pk
).limit(max(int(len(story_guids)*1.5), 10)))
existing_stories = dict((s.story_hash, s) for s in MStory.objects(
story_hash__in=story_hashes,
# story_date__gte=start_date,
# story_feed_id=self.feed.pk
))
ret_values = self.feed.add_update_stories(stories, existing_stories,
verbose=self.options['verbose'])