public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-08-31 21:41:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Refactoring the living crap out of the offline network operations. Now work in order and all cancelable with a single call. Still need to fix the network operation queue for downloading images and uploading the progress bar.

Browse source
This commit is contained in:
Samuel Clay 2013-07-15 18:25:09 -07:00
parent bcac92bcd7
commit 5745044927
10 changed files with 569 additions and 430 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/reader/views.py
View file

@ -952,7 +952,7 @@ def unread_story_hashes(request):
logging.user(request, "~FYLoading ~FCunread story hashes~FY: ~SB%s feeds~SN (%s story hashes)" %
(len(feed_ids), len(story_hashes)))
time.sleep(1)
return dict(unread_feed_story_hashes=story_hashes)
@ajax_login_required

22
clients/ios/Classes/NewsBlurAppDelegate.h
View file

@ -9,7 +9,6 @@
#import <UIKit/UIKit.h>
#import "BaseViewController.h"
#import "FMDatabaseQueue.h"
#import "ASINetworkQueue.h"
#define FEED_DETAIL_VIEW_TAG 1000001
#define STORY_DETAIL_VIEW_TAG 1000002
@ -138,10 +137,10 @@
NSMutableArray * dictFoldersArray;
FMDatabaseQueue *database;
NSOperationQueue *offlineQueue;
NSArray *categories;
NSDictionary *categoryFeeds;
UIImageView *splashView;
ASINetworkQueue *operationQueue;
NSMutableDictionary *activeCachedImages;
}
@ -186,7 +185,6 @@
@property (nonatomic, readwrite) BOOL isSocialRiverView;
@property (nonatomic, readwrite) BOOL isTryFeedView;
@property (nonatomic, readwrite) BOOL inFindingStoryMode;
@property (nonatomic, readwrite) BOOL hasQueuedReadStories;
@property (nonatomic) NSString *tryFeedStoryId;
@property (nonatomic) NSString *tryFeedCategory;
@property (nonatomic, readwrite) BOOL popoverHasFeedView;
@ -238,8 +236,9 @@
@property (nonatomic) NSArray *categories;
@property (nonatomic) NSDictionary *categoryFeeds;
@property (readwrite) FMDatabaseQueue *database;
@property (readwrite) ASINetworkQueue *operationQueue;
@property (nonatomic) NSOperationQueue *offlineQueue;
@property (nonatomic) NSMutableDictionary *activeCachedImages;
@property (nonatomic, readwrite) BOOL hasQueuedReadStories;
+ (NewsBlurAppDelegate*) sharedAppDelegate;
- (void)startupAnimationDone:(NSString *)animationID finished:(NSNumber *)finished context:(void *)context;
@ -340,19 +339,14 @@
- (int)databaseSchemaVersion:(FMDatabase *)db;
- (void)createDatabaseConnection;
- (void)setupDatabase:(FMDatabase *)db;
- (void)fetchUnreadHashes;
- (void)storeUnreadHashes:(ASIHTTPRequest *)request;
- (void)fetchAllUnreadStories;
- (void)storeAllUnreadStories:(ASIHTTPRequest *)request;
- (void)startOfflineQueue;
- (void)startOfflineFetchStories;
- (void)startOfflineFetchImages;
- (void)flushQueuedReadStories:(BOOL)forceCheck withCallback:(void(^)())callback;
- (void)syncQueuedReadStories:(FMDatabase *)db withStories:(NSDictionary *)hashes withCallback:(void(^)())callback;
- (void)deleteAllCachedImages;
- (NSArray *)uncachedImageUrls;
- (void)fetchAllUncachedImages;
- (void)storeCachedImage:(ASIHTTPRequest *)request;
- (void)cachedImageQueueFinished:(ASINetworkQueue *)queue;
- (void)flushOldCachedImages;
- (void)prepareActiveCachedImages:(FMDatabase *)db;
- (void)flushOldCachedImages;
- (void)deleteAllCachedImages;
@end

495
clients/ios/Classes/NewsBlurAppDelegate.m
View file

@ -44,6 +44,9 @@
#import "FMDatabaseAdditions.h"
#import "JSON.h"
#import "IASKAppSettingsViewController.h"
#import "OfflineSyncUnreads.h"
#import "OfflineFetchStories.h"
#import "OfflineFetchImages.h"
@implementation NewsBlurAppDelegate
@ -94,7 +97,6 @@
@synthesize isTryFeedView;
@synthesize inFindingStoryMode;
@synthesize hasQueuedReadStories;
@synthesize tryFeedStoryId;
@synthesize tryFeedCategory;
@synthesize popoverHasFeedView;
@ -146,8 +148,9 @@
@synthesize database;
@synthesize categories;
@synthesize categoryFeeds;
@synthesize operationQueue;
@synthesize activeCachedImages;
@synthesize hasQueuedReadStories;
@synthesize offlineQueue;
+ (NewsBlurAppDelegate*) sharedAppDelegate {
return (NewsBlurAppDelegate*) [UIApplication sharedApplication].delegate;
@ -2224,8 +2227,34 @@
NSLog(@"Create db %d: %@", [db lastErrorCode], [db lastErrorMessage]);
}
- (void)startOfflineQueue {
if (!offlineQueue) {
offlineQueue = [NSOperationQueue new];
}
offlineQueue.name = @"Offline Queue";
[offlineQueue cancelAllOperations];
[offlineQueue setMaxConcurrentOperationCount:1];
OfflineSyncUnreads *operationSyncUnreads = [[OfflineSyncUnreads alloc] init];
[offlineQueue addOperation:operationSyncUnreads];
}
- (void)startOfflineFetchStories {
OfflineFetchStories *operationFetchStories = [[OfflineFetchStories alloc] init];
[offlineQueue addOperation:operationFetchStories];
}
- (void)startOfflineFetchImages {
OfflineFetchImages *operationFetchImages = [[OfflineFetchImages alloc] init];
[offlineQueue addOperation:operationFetchImages];
}
- (void)flushQueuedReadStories:(BOOL)forceCheck withCallback:(void(^)())callback {
if (hasQueuedReadStories || forceCheck) {
if (self.hasQueuedReadStories || forceCheck) {
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT,
(unsigned long)NULL), ^(void) {
[self flushOldCachedImages];
@ -2242,7 +2271,7 @@
}
if ([[hashes allKeys] count]) {
hasQueuedReadStories = NO;
self.hasQueuedReadStories = NO;
[self syncQueuedReadStories:db withStories:hashes withCallback:callback];
} else {
if (callback) callback();
@ -2274,419 +2303,12 @@
}];
[request setFailedBlock:^{
NSLog(@"Failed mark read queued.");
hasQueuedReadStories = YES;
self.hasQueuedReadStories = YES;
if (callback) callback();
}];
[request startAsynchronous];
}
- (void)fetchUnreadHashes {
NSURL *url = [NSURL URLWithString:[NSString stringWithFormat:@"%@/reader/unread_story_hashes?include_timestamps=true",
NEWSBLUR_URL]];
ASIHTTPRequest *_request = [ASIHTTPRequest requestWithURL:url];
__weak ASIHTTPRequest *request = _request;
[request setResponseEncoding:NSUTF8StringEncoding];
[request setDefaultResponseEncoding:NSUTF8StringEncoding];
[request setFailedBlock:^(void) {
NSLog(@"Failed fetch all story hashes.");
}];
[request setCompletionBlock:^(void) {
[self storeUnreadHashes:request];
}];
[request setTimeOutSeconds:30];
[request startAsynchronous];
dispatch_async(dispatch_get_main_queue(), ^{
[self.feedsViewController showSyncingNotifier];
});
}
- (void)storeUnreadHashes:(ASIHTTPRequest *)request {
NSString *responseString = [request responseString];
NSData *responseData=[responseString dataUsingEncoding:NSUTF8StringEncoding];
NSError *error;
NSDictionary *results = [NSJSONSerialization
JSONObjectWithData:responseData
options:kNilOptions
error:&error];
__block __typeof__(self) _self = self;
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT,
(unsigned long)NULL), ^(void) {
[_self.database inTransaction:^(FMDatabase *db, BOOL *rollback) {
[db executeUpdate:@"DROP TABLE unread_hashes"];
[_self setupDatabase:db];
NSDictionary *hashes = [results objectForKey:@"unread_feed_story_hashes"];
for (NSString *feed in [hashes allKeys]) {
NSArray *story_hashes = [hashes objectForKey:feed];
for (NSArray *story_hash_tuple in story_hashes) {
[db executeUpdate:@"INSERT into unread_hashes"
"(story_feed_id, story_hash, story_timestamp) VALUES "
"(?, ?, ?)",
feed,
[story_hash_tuple objectAtIndex:0],
[story_hash_tuple objectAtIndex:1]
];
}
}
}];
[_self.database inTransaction:^(FMDatabase *db, BOOL *rollback) {
// Once all unread hashes are in, only keep under preference for offline limit
NSInteger offlineLimit = [[NSUserDefaults standardUserDefaults] integerForKey:@"offline_store_limit"];
NSString *order;
NSString *orderComp;
if ([[[NSUserDefaults standardUserDefaults] objectForKey:@"default_order"] isEqualToString:@"oldest"]) {
order = @"ASC";
orderComp = @">";
} else {
order = @"DESC";
orderComp = @"<";
}
FMResultSet *cursor = [db executeQuery:[NSString stringWithFormat:@"SELECT story_timestamp FROM unread_hashes ORDER BY story_timestamp %@ LIMIT 1 OFFSET %d", order, offlineLimit]];
int offlineLimitTimestamp = 0;
while ([cursor next]) {
offlineLimitTimestamp = [cursor intForColumn:@"story_timestamp"];
break;
}
NSLog(@"Deleting stories over limit: %d - %d", offlineLimit, offlineLimitTimestamp);
[db executeUpdate:[NSString stringWithFormat:@"DELETE FROM unread_hashes WHERE story_timestamp %@ %d", orderComp, offlineLimitTimestamp]];
}];
_self.totalUnfetchedStoryCount = 0;
_self.remainingUnfetchedStoryCount = 0;
_self.latestFetchedStoryDate = 0;
_self.totalUncachedImagesCount = 0;
_self.remainingUncachedImagesCount = 0;
[_self fetchAllUnreadStories];
});
}
- (NSArray *)unfetchedStoryHashes {
NSMutableArray *hashes = [NSMutableArray array];
[self.database inDatabase:^(FMDatabase *db) {
NSString *commonQuery = @"FROM unread_hashes u "
"LEFT OUTER JOIN stories s ON (s.story_hash = u.story_hash) "
"WHERE s.story_hash IS NULL";
int count = [db intForQuery:[NSString stringWithFormat:@"SELECT COUNT(1) %@", commonQuery]];
if (self.totalUnfetchedStoryCount == 0) {
self.totalUnfetchedStoryCount = count;
self.remainingUnfetchedStoryCount = self.totalUnfetchedStoryCount;
} else {
self.remainingUnfetchedStoryCount = count;
}
int limit = 100;
NSString *order;
if ([[[NSUserDefaults standardUserDefaults] objectForKey:@"default_order"] isEqualToString:@"oldest"]) {
order = @"ASC";
} else {
order = @"DESC";
}
FMResultSet *cursor = [db executeQuery:[NSString stringWithFormat:@"SELECT u.story_hash %@ ORDER BY u.story_timestamp %@ LIMIT %d", commonQuery, order, limit]];
while ([cursor next]) {
[hashes addObject:[cursor objectForColumnName:@"story_hash"]];
}
int start = (int)[[NSDate date] timeIntervalSince1970];
int end = self.latestFetchedStoryDate;
int seconds = start - (end ? end : start);
__block int hours = (int)round(seconds / 60.f / 60.f);
__block float progress = 0.f;
if (self.totalUnfetchedStoryCount) {
progress = 1.f - ((float)self.remainingUnfetchedStoryCount /
(float)self.totalUnfetchedStoryCount);
}
dispatch_async(dispatch_get_main_queue(), ^{
if (![[[NSUserDefaults standardUserDefaults]
objectForKey:@"offline_allowed"] boolValue]) {
dispatch_async(dispatch_get_main_queue(), ^{
[self.feedsViewController showDoneNotifier];
[self.feedsViewController hideNotifier];
});
return;
} else {
[self.feedsViewController showSyncingNotifier:progress hoursBack:hours];
}
});
}];
return hashes;
}
- (void)fetchAllUnreadStories {
NSArray *hashes = [self unfetchedStoryHashes];
if (![[[NSUserDefaults standardUserDefaults]
objectForKey:@"offline_allowed"] boolValue]) {
dispatch_async(dispatch_get_main_queue(), ^{
[self.feedsViewController hideNotifier];
});
return;
} else if ([hashes count] == 0) {
NSLog(@"Finished downloading unread stories. %d total", self.totalUnfetchedStoryCount);
dispatch_async(dispatch_get_main_queue(), ^{
if (![[[NSUserDefaults standardUserDefaults]
objectForKey:@"offline_image_download"] boolValue]) {
dispatch_async(dispatch_get_main_queue(), ^{
[self.feedsViewController showDoneNotifier];
[self.feedsViewController hideNotifier];
});
return;
} else {
[self.feedsViewController showCachingNotifier:0 hoursBack:1];
[self fetchAllUncachedImages];
}
});
return;
}
NSURL *url = [NSURL URLWithString:[NSString stringWithFormat:@"%@/reader/river_stories?page=0&h=%@",
NEWSBLUR_URL, [hashes componentsJoinedByString:@"&h="]]];
ASIHTTPRequest *_request = [ASIHTTPRequest requestWithURL:url];
__weak ASIHTTPRequest *request = _request;
[request setResponseEncoding:NSUTF8StringEncoding];
[request setDefaultResponseEncoding:NSUTF8StringEncoding];
[request setFailedBlock:^(void) {
NSLog(@"Failed fetch all unreads.");
}];
[request setCompletionBlock:^(void) {
[self storeAllUnreadStories:request];
}];
[request setTimeOutSeconds:30];
[request startAsynchronous];
}
- (void)storeAllUnreadStories:(ASIHTTPRequest *)request {
NSString *responseString = [request responseString];
NSData *responseData=[responseString dataUsingEncoding:NSUTF8StringEncoding];
NSError *error;
NSDictionary *results = [NSJSONSerialization
JSONObjectWithData:responseData
options:kNilOptions
error:&error];
__block BOOL anySuccess = NO;
__block __typeof__(self) _self = self;
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT,
(unsigned long)NULL), ^(void) {
[_self.database inTransaction:^(FMDatabase *db, BOOL *rollback) {
for (NSDictionary *story in [results objectForKey:@"stories"]) {
BOOL inserted = [db executeUpdate:@"INSERT into stories "
"(story_feed_id, story_hash, story_timestamp, story_json) VALUES "
"(?, ?, ?, ?)",
[story objectForKey:@"story_feed_id"],
[story objectForKey:@"story_hash"],
[story objectForKey:@"story_timestamp"],
[story JSONRepresentation]
];
if ([[story objectForKey:@"image_urls"] class] != [NSNull class] &&
[[story objectForKey:@"image_urls"] count]) {
for (NSString *imageUrl in [story objectForKey:@"image_urls"]) {
[db executeUpdate:@"INSERT INTO cached_images "
"(story_feed_id, story_hash, image_url) VALUES "
"(?, ?, ?)",
[story objectForKey:@"story_feed_id"],
[story objectForKey:@"story_hash"],
imageUrl
];
}
}
if (!anySuccess && inserted) anySuccess = YES;
}
if (anySuccess) {
_self.latestFetchedStoryDate = [[[[results objectForKey:@"stories"] lastObject]
objectForKey:@"story_timestamp"] intValue];
}
}];
if (anySuccess) {
[_self fetchAllUnreadStories];
} else {
dispatch_async(dispatch_get_main_queue(), ^{
[_self.feedsViewController hideNotifier];
});
}
});
}
#pragma mark -- Offline - Image Cache
- (NSArray *)uncachedImageUrls {
NSMutableArray *urls = [NSMutableArray array];
[self.database inDatabase:^(FMDatabase *db) {
NSString *commonQuery = @"FROM cached_images c "
"INNER JOIN unread_hashes u ON (c.story_hash = u.story_hash) "
"WHERE c.image_cached is null ";
int count = [db intForQuery:[NSString stringWithFormat:@"SELECT COUNT(1) %@", commonQuery]];
if (self.totalUncachedImagesCount == 0) {
self.totalUncachedImagesCount = count;
self.remainingUncachedImagesCount = self.totalUncachedImagesCount;
} else {
self.remainingUncachedImagesCount = count;
}
int limit = 96;
NSString *order;
if ([[[NSUserDefaults standardUserDefaults] objectForKey:@"default_order"] isEqualToString:@"oldest"]) {
order = @"ASC";
} else {
order = @"DESC";
}
NSString *sql = [NSString stringWithFormat:@"SELECT c.image_url, c.story_hash, u.story_timestamp %@ ORDER BY u.story_timestamp %@ LIMIT %d", commonQuery, order, limit];
FMResultSet *cursor = [db executeQuery:sql];
while ([cursor next]) {
[urls addObject:@[[cursor objectForColumnName:@"image_url"],
[cursor objectForColumnName:@"story_hash"],
[cursor objectForColumnName:@"story_timestamp"]]];
}
int start = (int)[[NSDate date] timeIntervalSince1970];
int end = [[[urls lastObject] objectAtIndex:2] intValue];
int seconds = start - (end ? end : start);
__block int hours = (int)round(seconds / 60.f / 60.f);
__block float progress = 0.f;
if (self.totalUncachedImagesCount) {
progress = 1.f - ((float)self.remainingUncachedImagesCount /
(float)self.totalUncachedImagesCount);
}
dispatch_async(dispatch_get_main_queue(), ^{
[self.feedsViewController showCachingNotifier:progress hoursBack:hours];
});
}];
return urls;
}
- (void)deleteAllCachedImages {
NSFileManager *fileManager = [[NSFileManager alloc] init];
NSError *error = nil;
NSArray *paths = NSSearchPathForDirectoriesInDomains(NSCachesDirectory, NSUserDomainMask, YES);
NSString *cacheDirectory = [[paths objectAtIndex:0] stringByAppendingPathComponent:@"story_images"];
NSArray *directoryContents = [fileManager contentsOfDirectoryAtPath:cacheDirectory error:&error];
int removed = 0;
if (error == nil) {
for (NSString *path in directoryContents) {
NSString *fullPath = [cacheDirectory stringByAppendingPathComponent:path];
BOOL removeSuccess = [fileManager removeItemAtPath:fullPath error:&error];
removed++;
if (!removeSuccess) {
continue;
}
}
}
NSLog(@"Deleted %d images.", removed);
}
- (void)fetchAllUncachedImages {
NSArray *urls = [self uncachedImageUrls];
operationQueue = [[ASINetworkQueue alloc] init];
operationQueue.maxConcurrentOperationCount = [[NSUserDefaults standardUserDefaults]
integerForKey:@"offline_image_concurrency"];
operationQueue.delegate = self;
if (![[[NSUserDefaults standardUserDefaults]
objectForKey:@"offline_image_download"] boolValue] ||
[urls count] == 0) {
NSLog(@"Finished caching images. %d total", self.totalUncachedImagesCount);
dispatch_async(dispatch_get_main_queue(), ^{
[self.feedsViewController showDoneNotifier];
[self.feedsViewController hideNotifier];
});
return;
}
for (NSArray *urlArray in urls) {
NSURL *url = [NSURL URLWithString:[urlArray objectAtIndex:0]];
NSString *storyHash = [urlArray objectAtIndex:1];
ASIHTTPRequest *request = [ASIHTTPRequest requestWithURL:url];
[request setUserInfo:@{@"story_hash": storyHash}];
[request setDelegate:self];
[request setDidFinishSelector:@selector(storeCachedImage:)];
[request setDidFailSelector:@selector(storeCachedImage:)];
[request setTimeOutSeconds:5];
[operationQueue addOperation:request];
}
[operationQueue setQueueDidFinishSelector:@selector(cachedImageQueueFinished:)];
[operationQueue setShouldCancelAllRequestsOnFailure:NO];
[operationQueue go];
// dispatch_async(dispatch_get_main_queue(), ^{
// [self.feedsViewController hideNotifier];
// });
}
- (void)storeCachedImage:(ASIHTTPRequest *)request {
NSString *storyHash = [[request userInfo] objectForKey:@"story_hash"];
if ([request responseStatusCode] == 200) {
NSData *responseData = [request responseData];
NSString *md5Url = [Utilities md5:[[request originalURL] absoluteString]];
NSLog(@"Storing image: %@ (%d bytes - %d in queue)", storyHash, [responseData length], [operationQueue requestsCount]);
if ([responseData length] <= 43) {
NSLog(@" ---> Image url: %@", [request url]);
}
NSFileManager *fileManager = [NSFileManager defaultManager];
NSArray *paths = NSSearchPathForDirectoriesInDomains(NSCachesDirectory, NSUserDomainMask, YES);
NSString *cacheDirectory = [[paths objectAtIndex:0] stringByAppendingPathComponent:@"story_images"];
NSString *fullPath = [cacheDirectory stringByAppendingPathComponent:[NSString stringWithFormat:@"%@", md5Url]];
[fileManager createFileAtPath:fullPath contents:responseData attributes:nil];
} else {
NSLog(@"Failed to fetch: %@ / %@", [[request originalURL] absoluteString], storyHash);
}
[self.database inDatabase:^(FMDatabase *db) {
[db executeUpdate:@"UPDATE cached_images SET "
"image_cached = 1 WHERE story_hash = ?",
storyHash];
}];
}
- (void)cachedImageQueueFinished:(ASINetworkQueue *)queue {
NSLog(@"Queue finished: %d total (%d remaining)", self.totalUncachedImagesCount, self.remainingUncachedImagesCount);
[self fetchAllUncachedImages];
// dispatch_async(dispatch_get_main_queue(), ^{
// [self.feedsViewController hideNotifier];
// });
}
- (void)flushOldCachedImages {
int deleted = 0;
NSFileManager *fileManager = [NSFileManager defaultManager];
NSArray *paths = NSSearchPathForDirectoriesInDomains(NSCachesDirectory, NSUserDomainMask, YES);
NSString *cacheDirectory = [[paths objectAtIndex:0] stringByAppendingPathComponent:@"story_images"];
NSDirectoryEnumerator* en = [fileManager enumeratorAtPath:cacheDirectory];
NSString* file;
while (file = [en nextObject])
{
NSError *error = nil;
NSString *filepath = [NSString stringWithFormat:[cacheDirectory stringByAppendingString:@"/%@"],file];
NSDate *creationDate = [[fileManager attributesOfItemAtPath:filepath error:nil] fileCreationDate];
NSDate *d = [[NSDate date] dateByAddingTimeInterval:-14*24*60*60];
NSDateFormatter *df = [[NSDateFormatter alloc] init]; // = [NSDateFormatter initWithDateFormat:@"yyyy-MM-dd"];
[df setDateFormat:@"EEEE d"];
if ([creationDate compare:d] == NSOrderedAscending) {
[[NSFileManager defaultManager]
removeItemAtPath:[cacheDirectory stringByAppendingPathComponent:file]
error:&error];
deleted += 1;
}
}
NSLog(@"Deleted %d old cached images", deleted);
}
- (void)prepareActiveCachedImages:(FMDatabase *)db {
activeCachedImages = [NSMutableDictionary dictionary];
NSDate *start = [NSDate date];
@ -2718,6 +2340,55 @@
NSLog(@"prepareActiveCachedImages time: %f", ([[NSDate date] timeIntervalSinceDate:start]));
}
- (void)flushOldCachedImages {
int deleted = 0;
NSFileManager *fileManager = [NSFileManager defaultManager];
NSArray *paths = NSSearchPathForDirectoriesInDomains(NSCachesDirectory, NSUserDomainMask, YES);
NSString *cacheDirectory = [[paths objectAtIndex:0] stringByAppendingPathComponent:@"story_images"];
NSDirectoryEnumerator* en = [fileManager enumeratorAtPath:cacheDirectory];
NSString* file;
while (file = [en nextObject])
{
NSError *error = nil;
NSString *filepath = [NSString stringWithFormat:[cacheDirectory stringByAppendingString:@"/%@"],file];
NSDate *creationDate = [[fileManager attributesOfItemAtPath:filepath error:nil] fileCreationDate];
NSDate *d = [[NSDate date] dateByAddingTimeInterval:-14*24*60*60];
NSDateFormatter *df = [[NSDateFormatter alloc] init]; // = [NSDateFormatter initWithDateFormat:@"yyyy-MM-dd"];
[df setDateFormat:@"EEEE d"];
if ([creationDate compare:d] == NSOrderedAscending) {
[[NSFileManager defaultManager]
removeItemAtPath:[cacheDirectory stringByAppendingPathComponent:file]
error:&error];
deleted += 1;
}
}
NSLog(@"Deleted %d old cached images", deleted);
}
- (void)deleteAllCachedImages {
NSFileManager *fileManager = [[NSFileManager alloc] init];
NSError *error = nil;
NSArray *paths = NSSearchPathForDirectoriesInDomains(NSCachesDirectory, NSUserDomainMask, YES);
NSString *cacheDirectory = [[paths objectAtIndex:0] stringByAppendingPathComponent:@"story_images"];
NSArray *directoryContents = [fileManager contentsOfDirectoryAtPath:cacheDirectory error:&error];
int removed = 0;
if (error == nil) {
for (NSString *path in directoryContents) {
NSString *fullPath = [cacheDirectory stringByAppendingPathComponent:path];
BOOL removeSuccess = [fileManager removeItemAtPath:fullPath error:&error];
removed++;
if (!removeSuccess) {
continue;
}
}
}
NSLog(@"Deleted %d images.", removed);
}
@end
#pragma mark -

5
clients/ios/Classes/NewsBlurViewController.m
View file

@ -748,7 +748,7 @@ static const CGFloat kFolderTitleHeight = 28;
if (self.inPullToRefresh_) {
self.inPullToRefresh_ = NO;
[self.appDelegate flushQueuedReadStories:YES withCallback:^{
[self.appDelegate fetchUnreadHashes];
[self.appDelegate startOfflineQueue];
}];
} else {
[self.appDelegate flushQueuedReadStories:YES withCallback:^{
@ -1533,7 +1533,6 @@ heightForHeaderInSection:(NSInteger)section {
[request startAsynchronous];
dispatch_async(dispatch_get_main_queue(), ^{
[self hideNotifier];
[self showCountingNotifier];
});
}
@ -1614,7 +1613,7 @@ heightForHeaderInSection:(NSInteger)section {
[appDelegate.folderCountCache removeAllObjects];
[self.feedTitlesTable reloadData];
[self refreshHeaderCounts];
[self.appDelegate fetchUnreadHashes];
[self.appDelegate startOfflineQueue];
}
// called when the date shown needs to be updated, optional

23
clients/ios/Classes/offline/OfflineFetchImages.h Normal file
View file

@ -0,0 +1,23 @@
//
// OfflineFetchImages.h
// NewsBlur
//
// Created by Samuel Clay on 7/15/13.
// Copyright (c) 2013 NewsBlur. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "NewsBlurAppDelegate.h"
#import "FMDatabaseQueue.h"
#import "ASINetworkQueue.h"
@interface OfflineFetchImages : NSOperation
@property (nonatomic) NewsBlurAppDelegate *appDelegate;
@property (readwrite) ASINetworkQueue *imageDownloadOperationQueue;
- (NSArray *)uncachedImageUrls;
- (void)storeCachedImage:(ASIHTTPRequest *)request;
- (void)cachedImageQueueFinished:(ASINetworkQueue *)queue;
@end

157
clients/ios/Classes/offline/OfflineFetchImages.m Normal file
View file

@ -0,0 +1,157 @@
//
// OfflineFetchImages.m
// NewsBlur
//
// Created by Samuel Clay on 7/15/13.
// Copyright (c) 2013 NewsBlur. All rights reserved.
//
#import "OfflineFetchImages.h"
#import "NewsBlurAppDelegate.h"
#import "NewsBlurViewController.h"
#import "FMDatabase.h"
#import "FMDatabaseAdditions.h"
#import "Utilities.h"
@implementation OfflineFetchImages
@synthesize imageDownloadOperationQueue;
@synthesize appDelegate;
- (void)main {
[self fetchImages];
}
- (void)fetchImages {
if (self.isCancelled) return;
NSLog(@"Fetching images...");
appDelegate = [NewsBlurAppDelegate sharedAppDelegate];
NSArray *urls = [self uncachedImageUrls];
imageDownloadOperationQueue = [[ASINetworkQueue alloc] init];
imageDownloadOperationQueue.maxConcurrentOperationCount = [[NSUserDefaults standardUserDefaults]
integerForKey:@"offline_image_concurrency"];
imageDownloadOperationQueue.delegate = self;
if (![[[NSUserDefaults standardUserDefaults]
objectForKey:@"offline_image_download"] boolValue] ||
[urls count] == 0) {
NSLog(@"Finished caching images. %d total", appDelegate.totalUncachedImagesCount);
dispatch_async(dispatch_get_main_queue(), ^{
[appDelegate.feedsViewController showDoneNotifier];
[appDelegate.feedsViewController hideNotifier];
});
return;
}
NSMutableArray *downloadRequests = [NSMutableArray array];
for (NSArray *urlArray in urls) {
NSURL *url = [NSURL URLWithString:[urlArray objectAtIndex:0]];
NSString *storyHash = [urlArray objectAtIndex:1];
ASIHTTPRequest *request = [ASIHTTPRequest requestWithURL:url];
[request setUserInfo:@{@"story_hash": storyHash}];
[request setDelegate:self];
[request setDidFinishSelector:@selector(storeCachedImage:)];
[request setDidFailSelector:@selector(storeCachedImage:)];
[request setTimeOutSeconds:5];
[downloadRequests addObject:request];
}
[imageDownloadOperationQueue addOperations:downloadRequests waitUntilFinished:YES];
[imageDownloadOperationQueue setQueueDidFinishSelector:@selector(cachedImageQueueFinished:)];
[imageDownloadOperationQueue setShouldCancelAllRequestsOnFailure:NO];
[imageDownloadOperationQueue go];
// dispatch_async(dispatch_get_main_queue(), ^{
// [appDelegate.feedsViewController hideNotifier];
// });
}
- (NSArray *)uncachedImageUrls {
NSMutableArray *urls = [NSMutableArray array];
[appDelegate.database inDatabase:^(FMDatabase *db) {
NSString *commonQuery = @"FROM cached_images c "
"INNER JOIN unread_hashes u ON (c.story_hash = u.story_hash) "
"WHERE c.image_cached is null ";
int count = [db intForQuery:[NSString stringWithFormat:@"SELECT COUNT(1) %@", commonQuery]];
if (appDelegate.totalUncachedImagesCount == 0) {
appDelegate.totalUncachedImagesCount = count;
appDelegate.remainingUncachedImagesCount = appDelegate.totalUncachedImagesCount;
} else {
appDelegate.remainingUncachedImagesCount = count;
}
int limit = 96;
NSString *order;
if ([[[NSUserDefaults standardUserDefaults] objectForKey:@"default_order"] isEqualToString:@"oldest"]) {
order = @"ASC";
} else {
order = @"DESC";
}
NSString *sql = [NSString stringWithFormat:@"SELECT c.image_url, c.story_hash, u.story_timestamp %@ ORDER BY u.story_timestamp %@ LIMIT %d", commonQuery, order, limit];
FMResultSet *cursor = [db executeQuery:sql];
while ([cursor next]) {
[urls addObject:@[[cursor objectForColumnName:@"image_url"],
[cursor objectForColumnName:@"story_hash"],
[cursor objectForColumnName:@"story_timestamp"]]];
}
int start = (int)[[NSDate date] timeIntervalSince1970];
int end = [[[urls lastObject] objectAtIndex:2] intValue];
int seconds = start - (end ? end : start);
__block int hours = (int)round(seconds / 60.f / 60.f);
__block float progress = 0.f;
if (appDelegate.totalUncachedImagesCount) {
progress = 1.f - ((float)appDelegate.remainingUncachedImagesCount /
(float)appDelegate.totalUncachedImagesCount);
}
dispatch_async(dispatch_get_main_queue(), ^{
[appDelegate.feedsViewController showCachingNotifier:progress hoursBack:hours];
});
}];
return urls;
}
- (void)storeCachedImage:(ASIHTTPRequest *)request {
if (self.isCancelled) return;
NSString *storyHash = [[request userInfo] objectForKey:@"story_hash"];
if ([request responseStatusCode] == 200) {
NSData *responseData = [request responseData];
NSString *md5Url = [Utilities md5:[[request originalURL] absoluteString]];
NSLog(@"Storing image: %@ (%d bytes - %d in queue)", storyHash, [responseData length], [imageDownloadOperationQueue requestsCount]);
if ([responseData length] <= 43) {
NSLog(@" ---> Image url: %@", [request url]);
}
NSFileManager *fileManager = [NSFileManager defaultManager];
NSArray *paths = NSSearchPathForDirectoriesInDomains(NSCachesDirectory, NSUserDomainMask, YES);
NSString *cacheDirectory = [[paths objectAtIndex:0] stringByAppendingPathComponent:@"story_images"];
NSString *fullPath = [cacheDirectory stringByAppendingPathComponent:[NSString stringWithFormat:@"%@", md5Url]];
[fileManager createFileAtPath:fullPath contents:responseData attributes:nil];
} else {
NSLog(@"Failed to fetch: %@ / %@", [[request originalURL] absoluteString], storyHash);
}
[appDelegate.database inDatabase:^(FMDatabase *db) {
[db executeUpdate:@"UPDATE cached_images SET "
"image_cached = 1 WHERE story_hash = ?",
storyHash];
}];
}
- (void)cachedImageQueueFinished:(ASINetworkQueue *)queue {
NSLog(@"Queue finished: %d total (%d remaining)", appDelegate.totalUncachedImagesCount, appDelegate.remainingUncachedImagesCount);
[self fetchImages];
// dispatch_async(dispatch_get_main_queue(), ^{
// [self.feedsViewController hideNotifier];
// });
}
@end

19
clients/ios/Classes/offline/OfflineFetchStories.h Normal file
View file

@ -0,0 +1,19 @@
//
// OfflineFetchStories.h
// NewsBlur
//
// Created by Samuel Clay on 7/15/13.
// Copyright (c) 2013 NewsBlur. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "NewsBlurAppDelegate.h"
@interface OfflineFetchStories : NSOperation
@property (nonatomic) NewsBlurAppDelegate *appDelegate;
- (NSArray *)unfetchedStoryHashes;
- (void)storeAllUnreadStories:(NSDictionary *)results;
@end

159
clients/ios/Classes/offline/OfflineFetchStories.m Normal file
View file

@ -0,0 +1,159 @@
//
// OfflineFetchStories.m
// NewsBlur
//
// Created by Samuel Clay on 7/15/13.
// Copyright (c) 2013 NewsBlur. All rights reserved.
//
#import "OfflineFetchStories.h"
#import "NewsBlurAppDelegate.h"
#import "NewsBlurViewController.h"
#import "FMDatabase.h"
#import "FMDatabaseAdditions.h"
#import "AFJSONRequestOperation.h"
#import "JSON.h"
@implementation OfflineFetchStories
@synthesize appDelegate;
- (void)main {
[self fetchStories];
}
- (void)fetchStories {
if (self.isCancelled) return;
NSLog(@"Fetching Stories...");
appDelegate = [NewsBlurAppDelegate sharedAppDelegate];
NSArray *hashes = [self unfetchedStoryHashes];
if (![[[NSUserDefaults standardUserDefaults]
objectForKey:@"offline_allowed"] boolValue]) {
dispatch_async(dispatch_get_main_queue(), ^{
[appDelegate.feedsViewController hideNotifier];
});
return;
} else if ([hashes count] == 0) {
NSLog(@"Finished downloading unread stories. %d total", appDelegate.totalUnfetchedStoryCount);
dispatch_async(dispatch_get_main_queue(), ^{
if (![[[NSUserDefaults standardUserDefaults]
objectForKey:@"offline_image_download"] boolValue]) {
[appDelegate.feedsViewController showDoneNotifier];
[appDelegate.feedsViewController hideNotifier];
} else {
[appDelegate.feedsViewController showCachingNotifier:0 hoursBack:1];
[appDelegate startOfflineFetchImages];
}
});
return;
}
NSURL *url = [NSURL URLWithString:[NSString stringWithFormat:@"%@/reader/river_stories?page=0&h=%@",
NEWSBLUR_URL, [hashes componentsJoinedByString:@"&h="]]];
AFJSONRequestOperation *request = [AFJSONRequestOperation
JSONRequestOperationWithRequest:[NSURLRequest requestWithURL:url]
success:^(NSURLRequest *request, NSHTTPURLResponse *response, id JSON) {
[self storeAllUnreadStories:JSON];
} failure:^(NSURLRequest *request, NSHTTPURLResponse *response, NSError *error, id JSON) {
NSLog(@"Failed fetch all unreads.");
}];
request.successCallbackQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT,
(unsigned long)NULL);
[request start];
[request waitUntilFinished];
}
- (NSArray *)unfetchedStoryHashes {
NSMutableArray *hashes = [NSMutableArray array];
[appDelegate.database inDatabase:^(FMDatabase *db) {
NSString *commonQuery = @"FROM unread_hashes u "
"LEFT OUTER JOIN stories s ON (s.story_hash = u.story_hash) "
"WHERE s.story_hash IS NULL";
NSLog(@"Checking unfetched hashes...");
int count = [db intForQuery:[NSString stringWithFormat:@"SELECT COUNT(1) %@", commonQuery]];
if (appDelegate.totalUnfetchedStoryCount == 0) {
appDelegate.totalUnfetchedStoryCount = count;
appDelegate.remainingUnfetchedStoryCount = appDelegate.totalUnfetchedStoryCount;
} else {
appDelegate.remainingUnfetchedStoryCount = count;
}
int limit = 100;
NSString *order;
if ([[[NSUserDefaults standardUserDefaults] objectForKey:@"default_order"] isEqualToString:@"oldest"]) {
order = @"ASC";
} else {
order = @"DESC";
}
FMResultSet *cursor = [db executeQuery:[NSString stringWithFormat:@"SELECT u.story_hash %@ ORDER BY u.story_timestamp %@ LIMIT %d", commonQuery, order, limit]];
while ([cursor next]) {
[hashes addObject:[cursor objectForColumnName:@"story_hash"]];
}
int start = (int)[[NSDate date] timeIntervalSince1970];
int end = appDelegate.latestFetchedStoryDate;
int seconds = start - (end ? end : start);
__block int hours = (int)round(seconds / 60.f / 60.f);
__block float progress = 0.f;
if (appDelegate.totalUnfetchedStoryCount) {
progress = 1.f - ((float)appDelegate.remainingUnfetchedStoryCount /
(float)appDelegate.totalUnfetchedStoryCount);
}
dispatch_async(dispatch_get_main_queue(), ^{
[appDelegate.feedsViewController showSyncingNotifier:progress hoursBack:hours];
});
}];
return hashes;
}
- (void)storeAllUnreadStories:(NSDictionary *)results {
if (self.isCancelled) return;
__block BOOL anySuccess = NO;
[appDelegate.database inTransaction:^(FMDatabase *db, BOOL *rollback) {
for (NSDictionary *story in [results objectForKey:@"stories"]) {
BOOL inserted = [db executeUpdate:@"INSERT into stories "
"(story_feed_id, story_hash, story_timestamp, story_json) VALUES "
"(?, ?, ?, ?)",
[story objectForKey:@"story_feed_id"],
[story objectForKey:@"story_hash"],
[story objectForKey:@"story_timestamp"],
[story JSONRepresentation]
];
if ([[story objectForKey:@"image_urls"] class] != [NSNull class] &&
[[story objectForKey:@"image_urls"] count]) {
for (NSString *imageUrl in [story objectForKey:@"image_urls"]) {
[db executeUpdate:@"INSERT INTO cached_images "
"(story_feed_id, story_hash, image_url) VALUES "
"(?, ?, ?)",
[story objectForKey:@"story_feed_id"],
[story objectForKey:@"story_hash"],
imageUrl
];
}
}
if (!anySuccess && inserted) anySuccess = YES;
}
if (anySuccess) {
appDelegate.latestFetchedStoryDate = [[[[results objectForKey:@"stories"] lastObject]
objectForKey:@"story_timestamp"] intValue];
}
}];
if (anySuccess) {
[self fetchStories];
} else {
dispatch_async(dispatch_get_main_queue(), ^{
[appDelegate.feedsViewController hideNotifier];
});
}
}
@end

20
clients/ios/Classes/offline/OfflineSyncUnreads.h Normal file
View file

@ -0,0 +1,20 @@
//
// OfflineSyncUnreads.h
// NewsBlur
//
// Created by Samuel Clay on 7/15/13.
// Copyright (c) 2013 NewsBlur. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "NewsBlurAppDelegate.h"
#import "FMDatabaseQueue.h"
#import "ASINetworkQueue.h"
@interface OfflineSyncUnreads : NSOperation
@property (nonatomic) NewsBlurAppDelegate *appDelegate;
- (void)storeUnreadHashes:(NSDictionary *)results;
@end

97
clients/ios/Classes/offline/OfflineSyncUnreads.m Normal file
View file

@ -0,0 +1,97 @@
//
// OfflineSyncUnreads.m
// NewsBlur
//
// Created by Samuel Clay on 7/15/13.
// Copyright (c) 2013 NewsBlur. All rights reserved.
//
#import "OfflineSyncUnreads.h"
#import "NewsBlurAppDelegate.h"
#import "NewsBlurViewController.h"
#import "FMResultSet.h"
#import "FMDatabase.h"
#import "AFJSONRequestOperation.h"
@implementation OfflineSyncUnreads
@synthesize appDelegate;
- (void)main {
appDelegate = [NewsBlurAppDelegate sharedAppDelegate];
NSLog(@"Syncing Unreads...");
dispatch_async(dispatch_get_main_queue(), ^{
[appDelegate.feedsViewController showSyncingNotifier];
});
NSURL *url = [NSURL URLWithString:[NSString stringWithFormat:@"%@/reader/unread_story_hashes?include_timestamps=true",
NEWSBLUR_URL]];
AFJSONRequestOperation *request = [AFJSONRequestOperation
JSONRequestOperationWithRequest:[NSURLRequest requestWithURL:url]
success:^(NSURLRequest *request, NSHTTPURLResponse *response, id JSON) {
[self storeUnreadHashes:JSON];
} failure:^(NSURLRequest *request, NSHTTPURLResponse *response, NSError *error, id JSON) {
NSLog(@"Failed fetch all story hashes.");
}];
request.successCallbackQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT,
(unsigned long)NULL);
[request start];
[request waitUntilFinished];
}
- (void)storeUnreadHashes:(NSDictionary *)results {
if (self.isCancelled) return;
[appDelegate.database inTransaction:^(FMDatabase *db, BOOL *rollback) {
NSLog(@"Storing unread story hashes...");
[db executeUpdate:@"DROP TABLE unread_hashes"];
[appDelegate setupDatabase:db];
NSDictionary *hashes = [results objectForKey:@"unread_feed_story_hashes"];
for (NSString *feed in [hashes allKeys]) {
NSArray *story_hashes = [hashes objectForKey:feed];
for (NSArray *story_hash_tuple in story_hashes) {
[db executeUpdate:@"INSERT into unread_hashes"
"(story_feed_id, story_hash, story_timestamp) VALUES "
"(?, ?, ?)",
feed,
[story_hash_tuple objectAtIndex:0],
[story_hash_tuple objectAtIndex:1]
];
}
}
}];
[appDelegate.database inTransaction:^(FMDatabase *db, BOOL *rollback) {
// Once all unread hashes are in, only keep under preference for offline limit
NSInteger offlineLimit = [[NSUserDefaults standardUserDefaults] integerForKey:@"offline_store_limit"];
NSString *order;
NSString *orderComp;
if ([[[NSUserDefaults standardUserDefaults] objectForKey:@"default_order"] isEqualToString:@"oldest"]) {
order = @"ASC";
orderComp = @">";
} else {
order = @"DESC";
orderComp = @"<";
}
FMResultSet *cursor = [db executeQuery:[NSString stringWithFormat:@"SELECT story_timestamp FROM unread_hashes ORDER BY story_timestamp %@ LIMIT 1 OFFSET %d", order, offlineLimit]];
int offlineLimitTimestamp = 0;
while ([cursor next]) {
offlineLimitTimestamp = [cursor intForColumn:@"story_timestamp"];
break;
}
NSLog(@"Deleting stories over limit: %d - %d", offlineLimit, offlineLimitTimestamp);
[db executeUpdate:[NSString stringWithFormat:@"DELETE FROM unread_hashes WHERE story_timestamp %@ %d", orderComp, offlineLimitTimestamp]];
}];
appDelegate.totalUnfetchedStoryCount = 0;
appDelegate.remainingUnfetchedStoryCount = 0;
appDelegate.latestFetchedStoryDate = 0;
appDelegate.totalUncachedImagesCount = 0;
appDelegate.remainingUncachedImagesCount = 0;
[appDelegate startOfflineFetchStories];
NSLog(@"Done syncing Unreads...");
}
@end