Merge branch 'django1.6' into django1.7

* django1.6:
  Fixing DB Monitor.
  Updating User Agent for all fetchers.
  Ignoring VSCode.
  Checking the 'credit' field if author not found. Works for TechRepublic, which I suspect is a one off.
  Handling multiple newsletter feeds.
  Attempting a fix of stories taking too long to sort while counting unreads.
  Fixing newsletter search.
  Fixing newsletter search.
  Newsletters feeds now de-duplicate based on sender email as well as sender name. This should cut down on duplicates.
  Android v10.1.1.
  Ignoring cache for homepage user.
  Gotta recreate empty_values somehow
  Fix exception
  Update dependencies
  #1380 Use image proxy only for http images
  #1379 Automatic light/dark theme
  #1367 Starred stories service

.gitignore

@@ -11,6 +11,7 @@ media/iphone/build
 build/
 .DS_Store
 **/*.perspectivev*
+.vscode/*
 data/
 config/certificates
 **/*.xcuserstate

apps/newsletters/models.py

@@ -31,9 +31,26 @@ class EmailNewsletter:
             return
         usf.add_folder('', 'Newsletters')
         
+        # First look for the email address
         try:
             feed = Feed.objects.get(feed_address=feed_address)
+        except Feed.MultipleObjectsReturned:
+            feeds = Feed.objects.filter(feed_address=feed_address).limit(1)
+            if feeds.count():
+                feed = feeds[0]
         except Feed.DoesNotExist:
+            feed = None
+
+        # If not found, check among titles user has subscribed to
+        if not feed:
+            newsletter_subs = UserSubscription.objects.filter(user=user, feed__feed_address__contains="newsletter:").only('feed')
+            newsletter_feed_ids = [us.feed.pk for us in newsletter_subs]
+            feeds = Feed.objects.filter(feed_title__iexact=sender_name, pk__in=newsletter_feed_ids)
+            if feeds.count():
+                feed = feeds[0]
+        
+        # Create a new feed if it doesn't exist by sender name or email
+        if not feed:
             feed = Feed.objects.create(feed_address=feed_address, 
                                        feed_link='http://' + sender_domain,
                                        feed_title=sender_name,
@@ -148,8 +165,8 @@ class EmailNewsletter:
         
         return profile.user
     
-    def _feed_address(self, user, sender):
+    def _feed_address(self, user, sender_email):
-        return 'newsletter:%s:%s' % (user.pk, sender)
+        return 'newsletter:%s:%s' % (user.pk, sender_email)
     
     def _split_sender(self, sender):
         tokens = re.search('(.*?) <(.*?)@(.*?)>', sender)

apps/reader/models.py

@@ -769,6 +769,9 @@ class UserSubscription(models.Model):
                 except pymongo.errors.OperationFailure, e:
                     stories_db = MStory.objects(story_hash__in=unread_story_hashes)[:100]
                     stories = Feed.format_stories(stories_db, self.feed_id)
+                except pymongo.errors.OperationFailure, e:
+                    stories_db = MStory.objects(story_hash__in=unread_story_hashes)[:25]
+                    stories = Feed.format_stories(stories_db, self.feed_id)
                     
             unread_stories = []
             for story in stories:

apps/rss_feeds/icon_importer.py

@@ -275,14 +275,12 @@ class IconImporter(object):
         @timelimit(30)
         def _1(url):
             headers = {
-                'User-Agent': 'NewsBlur Favicon Fetcher - %s subscriber%s - %s '
+                'User-Agent': 'NewsBlur Favicon Fetcher - %s subscriber%s - %s %s' %
-                              '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
-                              'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
-                              'Safari/534.48.3)' %
                               (
                                   self.feed.num_subscribers,
                                   's' if self.feed.num_subscribers != 1 else '',
-                                  self.feed.permalink
+                                  self.feed.permalink,
+                                  self.feed.fake_user_agent,
                               ),
                 'Connection': 'close',
                 'Accept': 'image/png,image/x-icon,image/*;q=0.9,*/*;q=0.8'

apps/rss_feeds/models.py

@@ -1117,20 +1117,20 @@ class Feed(models.Model):
             # A known workaround is using facebook's user agent.
             return 'facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)'
 
-        ua = ('NewsBlur Feed Fetcher - %s subscriber%s - %s '
+        ua = ('NewsBlur Feed Fetcher - %s subscriber%s - %s %s' % (
-              '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) '
-              'AppleWebKit/537.36 (KHTML, like Gecko) '
-              'Chrome/56.0.2924.87 Safari/537.36)' % (
                    self.num_subscribers,
                    's' if self.num_subscribers != 1 else '',
                    self.permalink,
+                   self.fake_user_agent,
               ))
 
         return ua
     
     @property
     def fake_user_agent(self):
-        ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
+        ua = ('("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
+              'AppleWebKit/605.1.15 (KHTML, like Gecko) '
+              'Version/14.0.1 Safari/605.1.15")')
         
         return ua
     

apps/rss_feeds/page_importer.py

@@ -51,13 +51,11 @@ class PageImporter(object):
     @property
     def headers(self):
         return {
-            'User-Agent': 'NewsBlur Page Fetcher - %s subscriber%s - %s '
+            'User-Agent': 'NewsBlur Page Fetcher - %s subscriber%s - %s %s' % (
-                          '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
-                          'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
-                          'Safari/534.48.3)' % (
                 self.feed.num_subscribers,
                 's' if self.feed.num_subscribers != 1 else '',
                 self.feed.permalink,
+                self.feed.fake_user_agent,
             ),
         }
     

apps/rss_feeds/text_importer.py

@@ -37,13 +37,11 @@ class TextImporter:
     def headers(self):
         num_subscribers = getattr(self.feed, 'num_subscribers', 0)
         return {
-            'User-Agent': 'NewsBlur Content Fetcher - %s subscriber%s - %s '
+            'User-Agent': 'NewsBlur Content Fetcher - %s subscriber%s - %s %s' % (
-                          '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
-                          'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
-                          'Safari/534.48.3)' % (
                               num_subscribers,
                               's' if num_subscribers != 1 else '',
-                              getattr(self.feed, 'permalink', '')
+                              getattr(self.feed, 'permalink', ''),
+                              self.feed.fake_user_agent,
                           ),
         }
 

apps/social/models.py

@@ -2312,10 +2312,7 @@ class MSharedStory(mongo.DynamicDocument):
             return self.image_sizes
             
         headers = {
-            'User-Agent': 'NewsBlur Image Fetcher - %s '
+            'User-Agent': 'NewsBlur Image Fetcher - %s' % (
-                          '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
-                          'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
-                          'Safari/534.48.3)' % (
                 settings.NEWSBLUR_URL
             ),
         }

clients/android/NewsBlur/build.gradle

@@ -9,7 +9,7 @@ buildscript {
         google()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:4.0.2'
+        classpath 'com.android.tools.build:gradle:4.1.0'
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
     }
 }
@@ -29,15 +29,15 @@ apply plugin: 'checkstyle'
 
 dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
-    implementation 'androidx.fragment:fragment:1.2.5'
+    implementation 'androidx.fragment:fragment-ktx:1.2.5'
     implementation 'androidx.recyclerview:recyclerview:1.1.0'
     implementation 'androidx.swiperefreshlayout:swiperefreshlayout:1.1.0'
-    implementation 'com.squareup.okhttp3:okhttp:3.12.12'
+    implementation 'com.squareup.okhttp3:okhttp:4.9.0'
     implementation 'com.google.code.gson:gson:2.8.6'
-    implementation 'com.android.billingclient:billing:3.0.0'
+    implementation 'com.android.billingclient:billing:3.0.1'
     implementation 'nl.dionsegijn:konfetti:1.2.2'
     implementation 'com.github.jinatonic.confetti:confetti:1.1.2'
-    implementation 'com.google.android.play:core:1.8.2'
+    implementation 'com.google.android.play:core:1.8.3'
 }
 
 android {
@@ -46,8 +46,8 @@ android {
         applicationId "com.newsblur"
         minSdkVersion 21
         targetSdkVersion 29
-        versionCode 176
+        versionCode 177
-        versionName "10.1"
+        versionName "10.1.1"
     }
     compileOptions.with {
         sourceCompatibility = JavaVersion.VERSION_1_8

clients/android/NewsBlur/res/values/strings.xml

@@ -447,6 +447,7 @@
     <string name="sync_status_recounts">Catching up reading actions...</string>
     <string name="sync_status_ffsync">On its way...</string>
     <string name="sync_status_cleanup">Cleaning up...</string>
+    <string name="sync_status_starred">Sync saved stories actions…</string>
     <string name="sync_status_stories">Fetching fresh stories...</string>
     <string name="sync_status_unreads">Storing%sunread stories...</string>
     <string name="sync_status_text">Storing text for %s stories...</string>

clients/android/NewsBlur/src/com/newsblur/database/BlurDatabaseHelper.java

@@ -284,6 +284,19 @@ public class BlurDatabaseHelper {
         return hashes;
     }
 
+    public Set<String> getStarredStoryHashes() {
+        String q = "SELECT " + DatabaseConstants.STORY_HASH +
+                " FROM " + DatabaseConstants.STORY_TABLE +
+                " WHERE " + DatabaseConstants.STORY_STARRED + " = 1" ;
+        Cursor c = dbRO.rawQuery(q, null);
+        Set<String> hashes = new HashSet<>(c.getCount());
+        while (c.moveToNext()) {
+            hashes.add(c.getString(c.getColumnIndexOrThrow(DatabaseConstants.STORY_HASH)));
+        }
+        c.close();
+        return hashes;
+    }
+
     public Set<String> getAllStoryImages() {
         Cursor c = dbRO.query(DatabaseConstants.STORY_TABLE, new String[]{DatabaseConstants.STORY_IMAGE_URLS}, null, null, null, null, null);
         Set<String> urls = new HashSet<String>(c.getCount());
@@ -584,6 +597,22 @@ public class BlurDatabaseHelper {
         }
     }
 
+    public void markStoryHashesStarred(Collection<String> hashes, boolean isStarred) {
+        synchronized (RW_MUTEX) {
+            dbRW.beginTransaction();
+            try {
+                ContentValues values = new ContentValues();
+                values.put(DatabaseConstants.STORY_STARRED, isStarred);
+                for (String hash : hashes) {
+                    dbRW.update(DatabaseConstants.STORY_TABLE, values, DatabaseConstants.STORY_HASH + " = ?", new String[]{hash});
+                }
+                dbRW.setTransactionSuccessful();
+            } finally {
+                dbRW.endTransaction();
+            }
+        }
+    }
+
     public void setFeedsActive(Set<String> feedIds, boolean active) {
         synchronized (RW_MUTEX) {
             dbRW.beginTransaction();

clients/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java

@@ -587,8 +587,8 @@ public class FolderListFragment extends NbFragment implements OnCreateContextMen
 
 	private void checkAccountFeedsLimit() {
         new Handler().postDelayed(() -> {
-            if (adapter.totalActiveFeedCount > AppConstants.FREE_ACCOUNT_SITE_LIMIT && !PrefsUtils.getIsPremium(requireContext())) {
+            if (getActivity() != null && adapter.totalActiveFeedCount > AppConstants.FREE_ACCOUNT_SITE_LIMIT && !PrefsUtils.getIsPremium(getActivity())) {
-                Intent intent = new Intent(requireActivity(), MuteConfig.class);
+                Intent intent = new Intent(getActivity(), MuteConfig.class);
                 startActivity(intent);
             }
         }, 2000);

clients/android/NewsBlur/src/com/newsblur/network/APIConstants.java

@@ -52,6 +52,7 @@ public class APIConstants {
     public static final String PATH_MARK_STORY_AS_UNREAD = "/reader/mark_story_as_unread/";
     public static final String PATH_MARK_STORY_HASH_UNREAD = "/reader/mark_story_hash_as_unread/";
     public static final String PATH_STARRED_STORIES = "/reader/starred_stories";
+    public static final String PATH_STARRED_STORY_HASHES = "/reader/starred_story_hashes";
 	public static final String PATH_FEED_AUTOCOMPLETE = "/rss_feeds/feed_autocomplete";
 	public static final String PATH_LIKE_COMMENT = "/social/like_comment";
 	public static final String PATH_UNLIKE_COMMENT = "/social/remove_like_comment";

clients/android/NewsBlur/src/com/newsblur/network/APIManager.java

@@ -37,6 +37,7 @@ import com.newsblur.network.domain.LoginResponse;
 import com.newsblur.network.domain.NewsBlurResponse;
 import com.newsblur.network.domain.ProfileResponse;
 import com.newsblur.network.domain.RegisterResponse;
+import com.newsblur.network.domain.StarredStoryHashesResponse;
 import com.newsblur.network.domain.StoriesResponse;
 import com.newsblur.network.domain.StoryTextResponse;
 import com.newsblur.network.domain.UnreadCountResponse;
@@ -279,6 +280,11 @@ public class APIManager {
         return (UnreadStoryHashesResponse) response.getResponse(gson, UnreadStoryHashesResponse.class);
     }
 
+    public StarredStoryHashesResponse getStarredStoryHashes() {
+	    APIResponse response = get(buildUrl(APIConstants.PATH_STARRED_STORY_HASHES));
+	    return response.getResponse(gson, StarredStoryHashesResponse.class);
+    }
+
     public StoriesResponse getStoriesByHash(List<String> storyHashes) {
 		ValueMultimap values = new ValueMultimap();
         for (String hash : storyHashes) {

clients/android/NewsBlur/src/com/newsblur/network/domain/StarredStoryHashesResponse.kt

@@ -0,0 +1,7 @@
+package com.newsblur.network.domain
+
+import com.google.gson.annotations.SerializedName
+
+data class StarredStoryHashesResponse(
+        @SerializedName("starred_story_hashes")
+        val starredStoryHashes: Set<String> = HashSet()) : NewsBlurResponse()

clients/android/NewsBlur/src/com/newsblur/serialization/StoryTypeAdapter.java

@@ -24,7 +24,7 @@ public class StoryTypeAdapter implements JsonDeserializer<Story> {
 
     // any characters we don't want in the short description, such as newlines or placeholders
     private final static Pattern ShortContentExcludes = Pattern.compile("[\\uFFFC\\u000A\\u000B\\u000C\\u000D]");
-    private final static Pattern httpSniff = Pattern.compile("(?:http):\\/\\/");
+    private final static Pattern httpSniff = Pattern.compile("(?:http):\\//");
 
     public StoryTypeAdapter() {
         this.gson = new GsonBuilder()
@@ -43,9 +43,11 @@ public class StoryTypeAdapter implements JsonDeserializer<Story> {
 
         // replace http image urls with https
         if (httpSniff.matcher(story.content).find() && story.secureImageUrls != null && story.secureImageUrls.size() > 0) {
-            for (String httpUrl : story.secureImageUrls.keySet()) {
+            for (String url : story.secureImageUrls.keySet()) {
-                String httpsUrl = story.secureImageUrls.get(httpUrl);
+                if (httpSniff.matcher(url).find()) {
-                story.content = story.content.replace(httpUrl, httpsUrl);
+                    String httpsUrl = story.secureImageUrls.get(url);
+                    story.content = story.content.replace(url, httpsUrl);
+                }
             }
         }
         

clients/android/NewsBlur/src/com/newsblur/service/NBSyncService.java

@@ -133,6 +133,7 @@ public class NBSyncService extends JobService {
     private List<JobParameters> outstandingStartParams = new ArrayList<JobParameters>();
     private boolean mainSyncRunning = false;
     CleanupService cleanupService;
+    StarredService starredService;
     OriginalTextService originalTextService;
     UnreadsService unreadsService;
     ImagePrefetchService imagePrefetchService;
@@ -166,6 +167,7 @@ public class NBSyncService extends JobService {
             dbHelper = new BlurDatabaseHelper(this);
             iconCache = FileCache.asIconCache(this);
             cleanupService = new CleanupService(this);
+            starredService = new StarredService(this);
             originalTextService = new OriginalTextService(this);
             unreadsService = new UnreadsService(this);
             imagePrefetchService = new ImagePrefetchService(this);
@@ -612,6 +614,7 @@ public class NBSyncService extends JobService {
             UnreadsService.doMetadata();
             unreadsService.start();
             cleanupService.start();
+            starredService.start();
 
         } finally {
             FFSyncRunning = false;
@@ -952,6 +955,7 @@ public class NBSyncService extends JobService {
         //Log.d(this, "checking completion");
         if (mainSyncRunning) return;
         if ((cleanupService != null) && cleanupService.isRunning()) return;
+        if ((starredService != null) && starredService.isRunning()) return;
         if ((originalTextService != null) && originalTextService.isRunning()) return;
         if ((unreadsService != null) && unreadsService.isRunning()) return;
         if ((imagePrefetchService != null) && imagePrefetchService.isRunning()) return;
@@ -1036,6 +1040,7 @@ public class NBSyncService extends JobService {
         if (HousekeepingRunning) return context.getResources().getString(R.string.sync_status_housekeeping);
         if (FFSyncRunning) return context.getResources().getString(R.string.sync_status_ffsync);
         if (CleanupService.activelyRunning) return context.getResources().getString(R.string.sync_status_cleanup);
+        if (StarredService.activelyRunning) return context.getResources().getString(R.string.sync_status_starred);
         if (brief && !AppConstants.VERBOSE_LOG) return null;
         if (ActionsRunning) return String.format(context.getResources().getString(R.string.sync_status_actions), lastActionCount);
         if (RecountsRunning) return context.getResources().getString(R.string.sync_status_recounts);
@@ -1196,6 +1201,7 @@ public class NBSyncService extends JobService {
             }
             if (cleanupService != null) cleanupService.shutdown();
             if (unreadsService != null) unreadsService.shutdown();
+            if (starredService != null) starredService.shutdown();
             if (originalTextService != null) originalTextService.shutdown();
             if (imagePrefetchService != null) imagePrefetchService.shutdown();
             if (primaryExecutor != null) {

clients/android/NewsBlur/src/com/newsblur/service/StarredService.kt

@@ -0,0 +1,37 @@
+package com.newsblur.service
+
+class StarredService(parent: NBSyncService) : SubService(parent) {
+
+    companion object {
+        @JvmField
+        var activelyRunning = false
+    }
+
+    override fun exec() {
+        activelyRunning = true
+
+        if (parent.stopSync()) return
+
+        // get all starred story hashes from remote db
+        val starredHashesResponse = parent.apiManager.starredStoryHashes
+
+        if (parent.stopSync()) return
+
+        // get all starred story hashes from local db
+        val localStoryHashes = parent.dbHelper.starredStoryHashes
+
+        if (parent.stopSync()) return
+
+        val newStarredHashes = starredHashesResponse.starredStoryHashes.minus(localStoryHashes)
+        val invalidStarredHashes = localStoryHashes.minus(starredHashesResponse.starredStoryHashes)
+
+        if (newStarredHashes.isNotEmpty()) {
+            parent.dbHelper.markStoryHashesStarred(newStarredHashes, true)
+        }
+        if (invalidStarredHashes.isNotEmpty()) {
+            parent.dbHelper.markStoryHashesStarred(invalidStarredHashes, false)
+        }
+
+        activelyRunning = false
+    }
+}

clients/android/NewsBlur/src/com/newsblur/util/PrefsUtils.java

@@ -792,7 +792,7 @@ public class PrefsUtils {
 
     public static ThemeValue getSelectedTheme(Context context) {
         SharedPreferences prefs = context.getSharedPreferences(PrefConstants.PREFERENCES, 0);
-        String value = prefs.getString(PrefConstants.THEME, ThemeValue.LIGHT.name());
+        String value = prefs.getString(PrefConstants.THEME, ThemeValue.AUTO.name());
         // check for legacy hard-coded values. this can go away once installs of v152 or earlier are minimized
         if (value.equals("light")) {    
             setSelectedTheme(context, ThemeValue.LIGHT);

config/haproxy.conf.template

@@ -127,7 +127,7 @@ backend postgres
   
 backend mongo
   option httpchk GET /db_check/mongo
-  server mongo-db20         db20:5000               check inter 2000ms
+  server mongo-db20e        db20e:5000              check inter 2000ms
   server mongo-db23a        db23a:5000              check inter 2000ms
   server mongo-db25a        db25a:5000              check inter 2000ms
   server mongo-db30         db30:5000               check inter 2000ms

flask/db_monitor.py

@@ -1,7 +1,10 @@
 from flask import Flask, abort
 import flask_settings as settings
 import psycopg2
-import MySQLdb
+try:
+    import MySQLdb
+except ImportError:
+    pass
 import pymongo
 import redis
 import pyes

utils/story_functions.py

@@ -191,6 +191,9 @@ def pre_process_story(entry, encoding):
         
     entry['title'] = strip_tags(entry.get('title'))
     entry['author'] = strip_tags(entry.get('author'))
+    if not entry['author']:
+        entry['author'] = strip_tags(entry.get('credit'))
+
     
     entry['story_content'] = attach_media_scripts(entry['story_content'])
     

utils/user_functions.py

@@ -90,15 +90,12 @@ def get_user(request):
                 pass
         
     if user.is_anonymous():
-        user = cache.get('user:%s' % settings.HOMEPAGE_USERNAME, None)
+        try:
-        if not user:
+            user = User.objects.get(username=settings.HOMEPAGE_USERNAME)
-            try:
+        except User.DoesNotExist:
-                user = User.objects.get(username=settings.HOMEPAGE_USERNAME)
+            user = User.objects.create(username=settings.HOMEPAGE_USERNAME)
-                cache.set('user:%s' % user, user)
+            user.set_password('')
-            except User.DoesNotExist:
+            user.save()
-                user = User.objects.create(username=settings.HOMEPAGE_USERNAME)
-                user.set_password('')
-                user.save()
     return user
     
 def invalidate_template_cache(fragment_name, *variables):