Merge branch 'master' into django1.11

* master:
  Checking the 'credit' field if author not found. Works for TechRepublic, which I suspect is a one off.
  Handling multiple newsletter feeds.
  Attempting a fix of stories taking too long to sort while counting unreads.
  Fixing newsletter search.
  Fixing newsletter search.
  Newsletters feeds now de-duplicate based on sender email as well as sender name. This should cut down on duplicates.
  Android v10.1.1.
  Fix exception
  Update dependencies
  #1380 Use image proxy only for http images
  #1379 Automatic light/dark theme
  #1367 Starred stories service

apps/newsletters/models.py

@@ -31,9 +31,26 @@ class EmailNewsletter:
             return
         usf.add_folder('', 'Newsletters')
         
+        # First look for the email address
         try:
             feed = Feed.objects.get(feed_address=feed_address)
+        except Feed.MultipleObjectsReturned:
+            feeds = Feed.objects.filter(feed_address=feed_address).limit(1)
+            if feeds.count():
+                feed = feeds[0]
         except Feed.DoesNotExist:
+            feed = None
+
+        # If not found, check among titles user has subscribed to
+        if not feed:
+            newsletter_subs = UserSubscription.objects.filter(user=user, feed__feed_address__contains="newsletter:").only('feed')
+            newsletter_feed_ids = [us.feed.pk for us in newsletter_subs]
+            feeds = Feed.objects.filter(feed_title__iexact=sender_name, pk__in=newsletter_feed_ids)
+            if feeds.count():
+                feed = feeds[0]
+        
+        # Create a new feed if it doesn't exist by sender name or email
+        if not feed:
             feed = Feed.objects.create(feed_address=feed_address, 
                                        feed_link='http://' + sender_domain,
                                        feed_title=sender_name,
@@ -148,8 +165,8 @@ class EmailNewsletter:
         
         return profile.user
     
-    def _feed_address(self, user, sender):
+    def _feed_address(self, user, sender_email):
-        return 'newsletter:%s:%s' % (user.pk, sender)
+        return 'newsletter:%s:%s' % (user.pk, sender_email)
     
     def _split_sender(self, sender):
         tokens = re.search('(.*?) <(.*?)@(.*?)>', sender)

apps/reader/models.py

@@ -771,6 +771,9 @@ class UserSubscription(models.Model):
                 except pymongo.errors.OperationFailure, e:
                     stories_db = MStory.objects(story_hash__in=unread_story_hashes)[:100]
                     stories = Feed.format_stories(stories_db, self.feed_id)
+                except pymongo.errors.OperationFailure, e:
+                    stories_db = MStory.objects(story_hash__in=unread_story_hashes)[:25]
+                    stories = Feed.format_stories(stories_db, self.feed_id)
                     
             unread_stories = []
             for story in stories:

clients/android/NewsBlur/build.gradle

@@ -9,7 +9,7 @@ buildscript {
         google()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:4.0.2'
+        classpath 'com.android.tools.build:gradle:4.1.0'
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
     }
 }
@@ -29,15 +29,15 @@ apply plugin: 'checkstyle'
 
 dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
-    implementation 'androidx.fragment:fragment:1.2.5'
+    implementation 'androidx.fragment:fragment-ktx:1.2.5'
     implementation 'androidx.recyclerview:recyclerview:1.1.0'
     implementation 'androidx.swiperefreshlayout:swiperefreshlayout:1.1.0'
-    implementation 'com.squareup.okhttp3:okhttp:3.12.12'
+    implementation 'com.squareup.okhttp3:okhttp:4.9.0'
     implementation 'com.google.code.gson:gson:2.8.6'
-    implementation 'com.android.billingclient:billing:3.0.0'
+    implementation 'com.android.billingclient:billing:3.0.1'
     implementation 'nl.dionsegijn:konfetti:1.2.2'
     implementation 'com.github.jinatonic.confetti:confetti:1.1.2'
-    implementation 'com.google.android.play:core:1.8.2'
+    implementation 'com.google.android.play:core:1.8.3'
 }
 
 android {
@@ -46,8 +46,8 @@ android {
         applicationId "com.newsblur"
         minSdkVersion 21
         targetSdkVersion 29
-        versionCode 176
+        versionCode 177
-        versionName "10.1"
+        versionName "10.1.1"
     }
     compileOptions.with {
         sourceCompatibility = JavaVersion.VERSION_1_8

clients/android/NewsBlur/res/values/strings.xml

@@ -447,6 +447,7 @@
     <string name="sync_status_recounts">Catching up reading actions...</string>
     <string name="sync_status_ffsync">On its way...</string>
     <string name="sync_status_cleanup">Cleaning up...</string>
+    <string name="sync_status_starred">Sync saved stories actions…</string>
     <string name="sync_status_stories">Fetching fresh stories...</string>
     <string name="sync_status_unreads">Storing%sunread stories...</string>
     <string name="sync_status_text">Storing text for %s stories...</string>

clients/android/NewsBlur/src/com/newsblur/database/BlurDatabaseHelper.java

@@ -284,6 +284,19 @@ public class BlurDatabaseHelper {
         return hashes;
     }
 
+    public Set<String> getStarredStoryHashes() {
+        String q = "SELECT " + DatabaseConstants.STORY_HASH +
+                " FROM " + DatabaseConstants.STORY_TABLE +
+                " WHERE " + DatabaseConstants.STORY_STARRED + " = 1" ;
+        Cursor c = dbRO.rawQuery(q, null);
+        Set<String> hashes = new HashSet<>(c.getCount());
+        while (c.moveToNext()) {
+            hashes.add(c.getString(c.getColumnIndexOrThrow(DatabaseConstants.STORY_HASH)));
+        }
+        c.close();
+        return hashes;
+    }
+
     public Set<String> getAllStoryImages() {
         Cursor c = dbRO.query(DatabaseConstants.STORY_TABLE, new String[]{DatabaseConstants.STORY_IMAGE_URLS}, null, null, null, null, null);
         Set<String> urls = new HashSet<String>(c.getCount());
@@ -584,6 +597,22 @@ public class BlurDatabaseHelper {
         }
     }
 
+    public void markStoryHashesStarred(Collection<String> hashes, boolean isStarred) {
+        synchronized (RW_MUTEX) {
+            dbRW.beginTransaction();
+            try {
+                ContentValues values = new ContentValues();
+                values.put(DatabaseConstants.STORY_STARRED, isStarred);
+                for (String hash : hashes) {
+                    dbRW.update(DatabaseConstants.STORY_TABLE, values, DatabaseConstants.STORY_HASH + " = ?", new String[]{hash});
+                }
+                dbRW.setTransactionSuccessful();
+            } finally {
+                dbRW.endTransaction();
+            }
+        }
+    }
+
     public void setFeedsActive(Set<String> feedIds, boolean active) {
         synchronized (RW_MUTEX) {
             dbRW.beginTransaction();

clients/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java

@@ -587,8 +587,8 @@ public class FolderListFragment extends NbFragment implements OnCreateContextMen
 
 	private void checkAccountFeedsLimit() {
         new Handler().postDelayed(() -> {
-            if (adapter.totalActiveFeedCount > AppConstants.FREE_ACCOUNT_SITE_LIMIT && !PrefsUtils.getIsPremium(requireContext())) {
+            if (getActivity() != null && adapter.totalActiveFeedCount > AppConstants.FREE_ACCOUNT_SITE_LIMIT && !PrefsUtils.getIsPremium(getActivity())) {
-                Intent intent = new Intent(requireActivity(), MuteConfig.class);
+                Intent intent = new Intent(getActivity(), MuteConfig.class);
                 startActivity(intent);
             }
         }, 2000);

clients/android/NewsBlur/src/com/newsblur/network/APIConstants.java

@@ -52,6 +52,7 @@ public class APIConstants {
     public static final String PATH_MARK_STORY_AS_UNREAD = "/reader/mark_story_as_unread/";
     public static final String PATH_MARK_STORY_HASH_UNREAD = "/reader/mark_story_hash_as_unread/";
     public static final String PATH_STARRED_STORIES = "/reader/starred_stories";
+    public static final String PATH_STARRED_STORY_HASHES = "/reader/starred_story_hashes";
 	public static final String PATH_FEED_AUTOCOMPLETE = "/rss_feeds/feed_autocomplete";
 	public static final String PATH_LIKE_COMMENT = "/social/like_comment";
 	public static final String PATH_UNLIKE_COMMENT = "/social/remove_like_comment";

clients/android/NewsBlur/src/com/newsblur/network/APIManager.java

@@ -37,6 +37,7 @@ import com.newsblur.network.domain.LoginResponse;
 import com.newsblur.network.domain.NewsBlurResponse;
 import com.newsblur.network.domain.ProfileResponse;
 import com.newsblur.network.domain.RegisterResponse;
+import com.newsblur.network.domain.StarredStoryHashesResponse;
 import com.newsblur.network.domain.StoriesResponse;
 import com.newsblur.network.domain.StoryTextResponse;
 import com.newsblur.network.domain.UnreadCountResponse;
@@ -279,6 +280,11 @@ public class APIManager {
         return (UnreadStoryHashesResponse) response.getResponse(gson, UnreadStoryHashesResponse.class);
     }
 
+    public StarredStoryHashesResponse getStarredStoryHashes() {
+	    APIResponse response = get(buildUrl(APIConstants.PATH_STARRED_STORY_HASHES));
+	    return response.getResponse(gson, StarredStoryHashesResponse.class);
+    }
+
     public StoriesResponse getStoriesByHash(List<String> storyHashes) {
 		ValueMultimap values = new ValueMultimap();
         for (String hash : storyHashes) {

clients/android/NewsBlur/src/com/newsblur/network/domain/StarredStoryHashesResponse.kt

@@ -0,0 +1,7 @@
+package com.newsblur.network.domain
+
+import com.google.gson.annotations.SerializedName
+
+data class StarredStoryHashesResponse(
+        @SerializedName("starred_story_hashes")
+        val starredStoryHashes: Set<String> = HashSet()) : NewsBlurResponse()

clients/android/NewsBlur/src/com/newsblur/serialization/StoryTypeAdapter.java

@@ -24,7 +24,7 @@ public class StoryTypeAdapter implements JsonDeserializer<Story> {
 
     // any characters we don't want in the short description, such as newlines or placeholders
     private final static Pattern ShortContentExcludes = Pattern.compile("[\\uFFFC\\u000A\\u000B\\u000C\\u000D]");
-    private final static Pattern httpSniff = Pattern.compile("(?:http):\\/\\/");
+    private final static Pattern httpSniff = Pattern.compile("(?:http):\\//");
 
     public StoryTypeAdapter() {
         this.gson = new GsonBuilder()
@@ -43,9 +43,11 @@ public class StoryTypeAdapter implements JsonDeserializer<Story> {
 
         // replace http image urls with https
         if (httpSniff.matcher(story.content).find() && story.secureImageUrls != null && story.secureImageUrls.size() > 0) {
-            for (String httpUrl : story.secureImageUrls.keySet()) {
+            for (String url : story.secureImageUrls.keySet()) {
-                String httpsUrl = story.secureImageUrls.get(httpUrl);
+                if (httpSniff.matcher(url).find()) {
-                story.content = story.content.replace(httpUrl, httpsUrl);
+                    String httpsUrl = story.secureImageUrls.get(url);
+                    story.content = story.content.replace(url, httpsUrl);
+                }
             }
         }
         

clients/android/NewsBlur/src/com/newsblur/service/NBSyncService.java

@@ -133,6 +133,7 @@ public class NBSyncService extends JobService {
     private List<JobParameters> outstandingStartParams = new ArrayList<JobParameters>();
     private boolean mainSyncRunning = false;
     CleanupService cleanupService;
+    StarredService starredService;
     OriginalTextService originalTextService;
     UnreadsService unreadsService;
     ImagePrefetchService imagePrefetchService;
@@ -166,6 +167,7 @@ public class NBSyncService extends JobService {
             dbHelper = new BlurDatabaseHelper(this);
             iconCache = FileCache.asIconCache(this);
             cleanupService = new CleanupService(this);
+            starredService = new StarredService(this);
             originalTextService = new OriginalTextService(this);
             unreadsService = new UnreadsService(this);
             imagePrefetchService = new ImagePrefetchService(this);
@@ -612,6 +614,7 @@ public class NBSyncService extends JobService {
             UnreadsService.doMetadata();
             unreadsService.start();
             cleanupService.start();
+            starredService.start();
 
         } finally {
             FFSyncRunning = false;
@@ -952,6 +955,7 @@ public class NBSyncService extends JobService {
         //Log.d(this, "checking completion");
         if (mainSyncRunning) return;
         if ((cleanupService != null) && cleanupService.isRunning()) return;
+        if ((starredService != null) && starredService.isRunning()) return;
         if ((originalTextService != null) && originalTextService.isRunning()) return;
         if ((unreadsService != null) && unreadsService.isRunning()) return;
         if ((imagePrefetchService != null) && imagePrefetchService.isRunning()) return;
@@ -1036,6 +1040,7 @@ public class NBSyncService extends JobService {
         if (HousekeepingRunning) return context.getResources().getString(R.string.sync_status_housekeeping);
         if (FFSyncRunning) return context.getResources().getString(R.string.sync_status_ffsync);
         if (CleanupService.activelyRunning) return context.getResources().getString(R.string.sync_status_cleanup);
+        if (StarredService.activelyRunning) return context.getResources().getString(R.string.sync_status_starred);
         if (brief && !AppConstants.VERBOSE_LOG) return null;
         if (ActionsRunning) return String.format(context.getResources().getString(R.string.sync_status_actions), lastActionCount);
         if (RecountsRunning) return context.getResources().getString(R.string.sync_status_recounts);
@@ -1196,6 +1201,7 @@ public class NBSyncService extends JobService {
             }
             if (cleanupService != null) cleanupService.shutdown();
             if (unreadsService != null) unreadsService.shutdown();
+            if (starredService != null) starredService.shutdown();
             if (originalTextService != null) originalTextService.shutdown();
             if (imagePrefetchService != null) imagePrefetchService.shutdown();
             if (primaryExecutor != null) {

clients/android/NewsBlur/src/com/newsblur/service/StarredService.kt

@@ -0,0 +1,37 @@
+package com.newsblur.service
+
+class StarredService(parent: NBSyncService) : SubService(parent) {
+
+    companion object {
+        @JvmField
+        var activelyRunning = false
+    }
+
+    override fun exec() {
+        activelyRunning = true
+
+        if (parent.stopSync()) return
+
+        // get all starred story hashes from remote db
+        val starredHashesResponse = parent.apiManager.starredStoryHashes
+
+        if (parent.stopSync()) return
+
+        // get all starred story hashes from local db
+        val localStoryHashes = parent.dbHelper.starredStoryHashes
+
+        if (parent.stopSync()) return
+
+        val newStarredHashes = starredHashesResponse.starredStoryHashes.minus(localStoryHashes)
+        val invalidStarredHashes = localStoryHashes.minus(starredHashesResponse.starredStoryHashes)
+
+        if (newStarredHashes.isNotEmpty()) {
+            parent.dbHelper.markStoryHashesStarred(newStarredHashes, true)
+        }
+        if (invalidStarredHashes.isNotEmpty()) {
+            parent.dbHelper.markStoryHashesStarred(invalidStarredHashes, false)
+        }
+
+        activelyRunning = false
+    }
+}

clients/android/NewsBlur/src/com/newsblur/util/PrefsUtils.java

@@ -792,7 +792,7 @@ public class PrefsUtils {
 
     public static ThemeValue getSelectedTheme(Context context) {
         SharedPreferences prefs = context.getSharedPreferences(PrefConstants.PREFERENCES, 0);
-        String value = prefs.getString(PrefConstants.THEME, ThemeValue.LIGHT.name());
+        String value = prefs.getString(PrefConstants.THEME, ThemeValue.AUTO.name());
         // check for legacy hard-coded values. this can go away once installs of v152 or earlier are minimized
         if (value.equals("light")) {    
             setSelectedTheme(context, ThemeValue.LIGHT);

utils/story_functions.py

@@ -191,6 +191,9 @@ def pre_process_story(entry, encoding):
         
     entry['title'] = strip_tags(entry.get('title'))
     entry['author'] = strip_tags(entry.get('author'))
+    if not entry['author']:
+        entry['author'] = strip_tags(entry.get('credit'))
+
     
     entry['story_content'] = attach_media_scripts(entry['story_content'])