Lots of offline progress: cancelling sync in the middle of one correctly, fixing bug around missing story hashes in sync, updating progress bar more often, fixing image caching bug that preventing caching from running at all.

apps/reader/views.py

@@ -658,9 +658,9 @@ def load_single_feed(request, feed_id):
     if not usersub:
         data.update(feed.canonical())
     
-    # if page <= 1:
+    if page <= 1:
-    #     import random
+        import random
-    #     time.sleep(random.randint(0, 1))
+        time.sleep(random.randint(4, 4))
 
     return data
 

clients/ios/Classes/FeedDetailViewController.m

@@ -891,14 +891,9 @@
     int score = [NewsBlurAppDelegate computeStoryScore:[story objectForKey:@"intelligence"]];
     cell.storyScore = score;
     
-    if (self.isOffline) {
+    if (!appDelegate.hasLoadedFeedDetail) {
-        BOOL read;
+        cell.isRead = ![[self.unreadStoryHashes objectForKey:[story objectForKey:@"story_hash"]] boolValue];
-        if ([appDelegate.activeReadFilter isEqualToString:@"all"]) {
+        NSLog(@"Offline: %d - %@ - %@ - %@", cell.isRead, [story objectForKey:@"story_title"], [story objectForKey:@"story_hash"], self.unreadStoryHashes);
-            read = ![[self.unreadStoryHashes objectForKey:[story objectForKey:@"story_hash"]] boolValue];
-        } else {
-            read = NO;
-        }
-        cell.isRead = read || ([[story objectForKey:@"read_status"] intValue] == 1);
     } else {
         cell.isRead = [[story objectForKey:@"read_status"] intValue] == 1;
     }

clients/ios/Classes/NewsBlurAppDelegate.h

@@ -118,6 +118,7 @@
     int totalUnfetchedStoryCount;
     int remainingUnfetchedStoryCount;
     int latestFetchedStoryDate;
+    int latestCachedImageDate;
     int totalUncachedImagesCount;
     int remainingUncachedImagesCount;
     NSMutableArray * recentlyReadStories;
@@ -217,6 +218,7 @@
 @property (readwrite) int totalUncachedImagesCount;
 @property (readwrite) int remainingUncachedImagesCount;
 @property (readwrite) int latestFetchedStoryDate;
+@property (readwrite) int latestCachedImageDate;
 @property (readwrite) NSInteger selectedIntelligence;
 @property (readwrite) NSMutableArray * recentlyReadStories;
 @property (readwrite) NSMutableSet * recentlyReadFeeds;
@@ -342,6 +344,7 @@
 - (int)databaseSchemaVersion:(FMDatabase *)db;
 - (void)createDatabaseConnection;
 - (void)setupDatabase:(FMDatabase *)db;
+- (void)cancelOfflineQueue;
 - (void)startOfflineQueue;
 - (void)startOfflineFetchStories;
 - (void)startOfflineFetchImages;

clients/ios/Classes/NewsBlurAppDelegate.m

@@ -50,7 +50,7 @@
 
 @implementation NewsBlurAppDelegate
 
-#define CURRENT_DB_VERSION 23
+#define CURRENT_DB_VERSION 25
 #define IS_IPHONE_5 ( fabs( ( double )[ [ UIScreen mainScreen ] bounds ].size.height - ( double )568 ) < DBL_EPSILON )
 
 @synthesize window;
@@ -124,6 +124,7 @@
 @synthesize totalUnfetchedStoryCount;
 @synthesize remainingUnfetchedStoryCount;
 @synthesize latestFetchedStoryDate;
+@synthesize latestCachedImageDate;
 @synthesize totalUncachedImagesCount;
 @synthesize remainingUncachedImagesCount;
 @synthesize originalStoryCount;
@@ -285,6 +286,7 @@
     self.totalUnfetchedStoryCount = 0;
     self.remainingUnfetchedStoryCount = 0;
     self.latestFetchedStoryDate = 0;
+    self.latestCachedImageDate = 0;
     self.totalUncachedImagesCount = 0;
     self.remainingUncachedImagesCount = 0;
     [self setRecentlyReadStories:[NSMutableArray array]];
@@ -2226,7 +2228,7 @@
                                    " story_feed_id number,"
                                    " story_hash varchar(24),"
                                    " image_url varchar(1024),"
-                                   " image_cached boolean"
+                                   " image_cached boolean,"
                                    " failed boolean"
                                    ")"];
     [db executeUpdate:createImagesTable];
@@ -2254,6 +2256,12 @@
     NSLog(@"Create db %d: %@", [db lastErrorCode], [db lastErrorMessage]);
 }
 
+- (void)cancelOfflineQueue {
+    if (offlineQueue) {
+        [offlineQueue cancelAllOperations];
+    }
+}
+
 - (void)startOfflineQueue {
     if (!offlineQueue) {
         offlineQueue = [NSOperationQueue new];
@@ -2340,8 +2348,8 @@
 
 - (void)prepareActiveCachedImages:(FMDatabase *)db {
     activeCachedImages = [NSMutableDictionary dictionary];
-    NSDate *start = [NSDate date];
     NSArray *feedIds;
+    int cached = 0;
     
     if (isRiverView) {
         feedIds = activeFolderFeeds;
@@ -2349,7 +2357,7 @@
         feedIds = @[[activeFeed objectForKey:@"id"]];
     }
     NSString *sql = [NSString stringWithFormat:@"SELECT c.image_url, c.story_hash FROM cached_images c "
-                     "WHERE c.image_cached = 1 AND c.story_feed_id in (%@)",
+                     "WHERE c.image_cached = 1 AND c.failed is null AND c.story_feed_id in (%@)",
                      [feedIds componentsJoinedByString:@","]];
     FMResultSet *cursor = [db executeQuery:sql];
     
@@ -2364,9 +2372,10 @@
         }
         [imageUrls addObject:[cursor objectForColumnName:@"image_url"]];
         [activeCachedImages setObject:imageUrls forKey:storyHash];
+        cached++;
     }
     
-    NSLog(@"prepareActiveCachedImages time: %f", ([[NSDate date] timeIntervalSinceDate:start]));
+    NSLog(@"Pre-cached %d images", cached);
 }
 
 - (void)flushOldCachedImages {

clients/ios/Classes/NewsBlurViewController.m

@@ -448,6 +448,7 @@ static const CGFloat kFolderTitleHeight = 28;
 
     self.lastUpdate = [NSDate date];
     [self showRefreshNotifier];
+    [appDelegate cancelOfflineQueue];
 }
 
 - (void)finishedWithError:(ASIHTTPRequest *)request {    
@@ -1535,6 +1536,8 @@ heightForHeaderInSection:(NSInteger)section {
     dispatch_async(dispatch_get_main_queue(), ^{
         [self showCountingNotifier];
     });
+    
+    [self.appDelegate cancelOfflineQueue];
 }
 
 - (void)finishRefreshingFeedList:(ASIHTTPRequest *)request {

clients/ios/Classes/offline/OfflineFetchImages.m

@@ -22,7 +22,6 @@
 
     while (YES) {
         BOOL fetched = [self fetchImages];
-        NSLog(@"Fetched: %d", fetched);
         if (!fetched) break;
     }
 }
@@ -34,7 +33,7 @@
         return NO;
     }
 
-    NSLog(@"Fetching images...");
+//    NSLog(@"Fetching images...");
     NSArray *urls = [self uncachedImageUrls];
 
     imageDownloadOperationQueue = [[ASINetworkQueue alloc] init];
@@ -86,7 +85,7 @@
     [appDelegate.database inDatabase:^(FMDatabase *db) {
         NSString *commonQuery = @"FROM cached_images c "
         "INNER JOIN unread_hashes u ON (c.story_hash = u.story_hash) "
-        "WHERE c.image_cached is null and c.failed != 1 ";
+        "WHERE c.image_cached is null ";
         int count = [db intForQuery:[NSString stringWithFormat:@"SELECT COUNT(1) %@", commonQuery]];
         if (appDelegate.totalUncachedImagesCount == 0) {
             appDelegate.totalUncachedImagesCount = count;
@@ -95,7 +94,7 @@
             appDelegate.remainingUncachedImagesCount = count;
         }
         
-        int limit = 96;
+        int limit = 36;
         NSString *order;
         if ([[[NSUserDefaults standardUserDefaults] objectForKey:@"default_order"] isEqualToString:@"oldest"]) {
             order = @"ASC";
@@ -110,24 +109,30 @@
              [cursor objectForColumnName:@"story_hash"],
              [cursor objectForColumnName:@"story_timestamp"]]];
         }
-        int start = (int)[[NSDate date] timeIntervalSince1970];
+        [self updateProgress];
-        int end = [[[urls lastObject] objectAtIndex:2] intValue];
-        int seconds = start - (end ? end : start);
-        __block int hours = (int)round(seconds / 60.f / 60.f);
-        
-        __block float progress = 0.f;
-        if (appDelegate.totalUncachedImagesCount) {
-            progress = 1.f - ((float)appDelegate.remainingUncachedImagesCount /
-                              (float)appDelegate.totalUncachedImagesCount);
-        }
-        dispatch_async(dispatch_get_main_queue(), ^{
-            [appDelegate.feedsViewController showCachingNotifier:progress hoursBack:hours];
-        });
     }];
     
     return urls;
 }
 
+- (void)updateProgress {
+    if (self.isCancelled) return;
+    
+    int start = (int)[[NSDate date] timeIntervalSince1970];
+    int end = appDelegate.latestCachedImageDate;
+    int seconds = start - (end ? end : start);
+    __block int hours = (int)round(seconds / 60.f / 60.f);
+    
+    __block float progress = 0.f;
+    if (appDelegate.totalUncachedImagesCount) {
+        progress = 1.f - ((float)appDelegate.remainingUncachedImagesCount /
+                          (float)appDelegate.totalUncachedImagesCount);
+    }
+    dispatch_async(dispatch_get_main_queue(), ^{
+        [appDelegate.feedsViewController showCachingNotifier:progress hoursBack:hours];
+    });
+}
+
 - (void)storeCachedImage:(ASIHTTPRequest *)request {
     if (self.isCancelled) {
         NSLog(@"Image cancelled.");
@@ -139,14 +144,12 @@
                                              (unsigned long)NULL), ^{
         
         NSString *storyHash = [[request userInfo] objectForKey:@"story_hash"];
+        int storyTimestamp = [[[request userInfo] objectForKey:@"story_timestamp"] intValue];
         
         if ([request responseStatusCode] == 200) {
             NSData *responseData = [request responseData];
             NSString *md5Url = [Utilities md5:[[request originalURL] absoluteString]];
-            NSLog(@"Storing image: %@ (%d bytes - %d in queue)", storyHash, [responseData length], [imageDownloadOperationQueue requestsCount]);
+//            NSLog(@"Storing image: %@ (%d bytes - %d in queue)", storyHash, [responseData length], [imageDownloadOperationQueue requestsCount]);
-            if ([responseData length] <= 43) {
-                NSLog(@" ---> Image url: %@", [request url]);
-            }
             
             NSFileManager *fileManager = [NSFileManager defaultManager];
             NSArray *paths = NSSearchPathForDirectoriesInDomains(NSCachesDirectory, NSUserDomainMask, YES);
@@ -163,6 +166,21 @@
              "image_cached = 1 WHERE story_hash = ?",
              storyHash];
         }];
+        
+        if ([[[NSUserDefaults standardUserDefaults] objectForKey:@"default_order"] isEqualToString:@"oldest"]) {
+            if (storyTimestamp > appDelegate.latestCachedImageDate) {
+                appDelegate.latestCachedImageDate = storyTimestamp;
+            }
+        } else {
+            if (storyTimestamp < appDelegate.latestCachedImageDate) {
+                appDelegate.latestCachedImageDate = storyTimestamp;
+            }
+        }
+        
+        appDelegate.remainingUncachedImagesCount--;
+        if (appDelegate.remainingUncachedImagesCount % 10 == 0) {
+            [self updateProgress];
+        }
     });
 }
 

clients/ios/Classes/offline/OfflineFetchStories.m

@@ -23,7 +23,6 @@
 
     while (YES) {
         BOOL fetched = [self fetchStories];
-        NSLog(@"Fetched: %d", fetched);
         if (!fetched) break;
     }
 }
@@ -43,7 +42,7 @@
         });
         return NO;
     }
-    NSLog(@"Fetching Stories...");
+//    NSLog(@"Fetching Stories...");
     
     NSArray *hashes = [self unfetchedStoryHashes];
     
@@ -106,35 +105,43 @@
         while ([cursor next]) {
             [hashes addObject:[cursor objectForColumnName:@"story_hash"]];
         }
-        int start = (int)[[NSDate date] timeIntervalSince1970];
-        int end = appDelegate.latestFetchedStoryDate;
-        int seconds = start - (end ? end : start);
-        __block int hours = (int)round(seconds / 60.f / 60.f);
         
-        __block float progress = 0.f;
+        [self updateProgress];
-        if (appDelegate.totalUnfetchedStoryCount) {
-            progress = 1.f - ((float)appDelegate.remainingUnfetchedStoryCount /
-                              (float)appDelegate.totalUnfetchedStoryCount);
-        }
-        dispatch_async(dispatch_get_main_queue(), ^{
-            [appDelegate.feedsViewController showSyncingNotifier:progress hoursBack:hours];
-        });
     }];
     
     return hashes;
 }
 
+- (void)updateProgress {
+    if (self.isCancelled) return;
+    
+    int start = (int)[[NSDate date] timeIntervalSince1970];
+    int end = appDelegate.latestFetchedStoryDate;
+    int seconds = start - (end ? end : start);
+    __block int hours = (int)round(seconds / 60.f / 60.f);
+    
+    __block float progress = 0.f;
+    if (appDelegate.totalUnfetchedStoryCount) {
+        progress = 1.f - ((float)appDelegate.remainingUnfetchedStoryCount /
+                          (float)appDelegate.totalUnfetchedStoryCount);
+    }
+    dispatch_async(dispatch_get_main_queue(), ^{
+        [appDelegate.feedsViewController showSyncingNotifier:progress hoursBack:hours];
+    });
+}
+
 - (void)storeAllUnreadStories:(NSDictionary *)results withHashes:(NSArray *)hashes {
     NSMutableArray *storyHashes = [hashes mutableCopy];
     [appDelegate.database inTransaction:^(FMDatabase *db, BOOL *rollback) {
         BOOL anyInserted = NO;
         for (NSDictionary *story in [results objectForKey:@"stories"]) {
+            NSString *storyTimestamp = [story objectForKey:@"story_timestamp"];
             BOOL inserted = [db executeUpdate:@"INSERT into stories "
                              "(story_feed_id, story_hash, story_timestamp, story_json) VALUES "
                              "(?, ?, ?, ?)",
                              [story objectForKey:@"story_feed_id"],
                              [story objectForKey:@"story_hash"],
-                             [story objectForKey:@"story_timestamp"],
+                             storyTimestamp,
                              [story JSONRepresentation]
                              ];
             if ([[story objectForKey:@"image_urls"] class] != [NSNull class] &&
@@ -153,6 +160,20 @@
                 anyInserted = YES;
                 [storyHashes removeObject:[story objectForKey:@"story_hash"]];
             }
+            if ([[[NSUserDefaults standardUserDefaults] objectForKey:@"default_order"] isEqualToString:@"oldest"]) {
+                if ([storyTimestamp intValue] > appDelegate.latestFetchedStoryDate) {
+                    appDelegate.latestFetchedStoryDate = [storyTimestamp intValue];
+                }
+            } else {
+                if ([storyTimestamp intValue] < appDelegate.latestFetchedStoryDate) {
+                    appDelegate.latestFetchedStoryDate = [storyTimestamp intValue];
+                }
+            }
+            appDelegate.remainingUnfetchedStoryCount--;
+            if (appDelegate.remainingUnfetchedStoryCount % 10 == 0) {
+                [self updateProgress];
+            }
+
         }
         if (anyInserted) {
             appDelegate.latestFetchedStoryDate = [[[[results objectForKey:@"stories"] lastObject]
@@ -160,8 +181,8 @@
         }
         if ([storyHashes count]) {
             NSLog(@"Failed to fetch stories: %@", storyHashes);
-            [db executeUpdate:[NSString stringWithFormat:@"DELTE FROM unread_hashes WHERE story_hash IN (%@)",
+            [db executeUpdate:[NSString stringWithFormat:@"DELETE FROM unread_hashes WHERE story_hash IN (\"%@\")",
-                               [storyHashes componentsJoinedByString:@", "]]];
+                               [storyHashes componentsJoinedByString:@"\",\" "]]];
         }
     }];
 }

clients/ios/NewsBlur_Prefix.pch

@@ -3,6 +3,16 @@
     #import <UIKit/UIKit.h>
     #import "Underscore.h"
 
+//#define DEBUG 1
+
+#ifdef DEBUG
+    #define NEWSBLUR_URL [NSString stringWithFormat:@"http://nb.local.com"]
+    #define NEWSBLUR_HOST [NSString stringWithFormat:@"nb.local.com"]
+#else
+    #define NEWSBLUR_URL [NSString stringWithFormat:@"https://www.newsblur.com"]
+    #define NEWSBLUR_HOST [NSString stringWithFormat:@"www.newsblur.com"]
+#endif
+
     #define _ Underscore
 
     #define UIColorFromRGB(rgbValue) [UIColor \
@@ -10,14 +20,7 @@
     green:((float)((rgbValue & 0xFF00) >> 8))/255.0 \
     blue:((float)(rgbValue & 0xFF))/255.0 alpha:1.0]
 
-//    #define BACKGROUND_REFRESH_SECONDS -5
     #define BACKGROUND_REFRESH_SECONDS -10*60
-
-    #define NEWSBLUR_URL [NSString stringWithFormat:@"http://nb.local.com"]
-    #define NEWSBLUR_HOST [NSString stringWithFormat:@"nb.local.com"]
-//    #define NEWSBLUR_URL [NSString stringWithFormat:@"https://www.newsblur.com"]
-//    #define NEWSBLUR_HOST [NSString stringWithFormat:@"www.newsblur.com"]
-
     #define NEWSBLUR_LINK_COLOR 0x405BA8
     #define NEWSBLUR_HIGHLIGHT_COLOR 0xd2e6fd