diff --git a/apps/newsletters/models.py b/apps/newsletters/models.py index 10b09d53f..f83133dac 100644 --- a/apps/newsletters/models.py +++ b/apps/newsletters/models.py @@ -31,9 +31,26 @@ class EmailNewsletter: return usf.add_folder('', 'Newsletters') + # First look for the email address try: feed = Feed.objects.get(feed_address=feed_address) + except Feed.MultipleObjectsReturned: + feeds = Feed.objects.filter(feed_address=feed_address).limit(1) + if feeds.count(): + feed = feeds[0] except Feed.DoesNotExist: + feed = None + + # If not found, check among titles user has subscribed to + if not feed: + newsletter_subs = UserSubscription.objects.filter(user=user, feed__feed_address__contains="newsletter:").only('feed') + newsletter_feed_ids = [us.feed.pk for us in newsletter_subs] + feeds = Feed.objects.filter(feed_title__iexact=sender_name, pk__in=newsletter_feed_ids) + if feeds.count(): + feed = feeds[0] + + # Create a new feed if it doesn't exist by sender name or email + if not feed: feed = Feed.objects.create(feed_address=feed_address, feed_link='http://' + sender_domain, feed_title=sender_name, @@ -148,8 +165,8 @@ class EmailNewsletter: return profile.user - def _feed_address(self, user, sender): - return 'newsletter:%s:%s' % (user.pk, sender) + def _feed_address(self, user, sender_email): + return 'newsletter:%s:%s' % (user.pk, sender_email) def _split_sender(self, sender): tokens = re.search('(.*?) <(.*?)@(.*?)>', sender) diff --git a/apps/reader/models.py b/apps/reader/models.py index af371f444..cccf518fd 100644 --- a/apps/reader/models.py +++ b/apps/reader/models.py @@ -771,6 +771,9 @@ class UserSubscription(models.Model): except pymongo.errors.OperationFailure, e: stories_db = MStory.objects(story_hash__in=unread_story_hashes)[:100] stories = Feed.format_stories(stories_db, self.feed_id) + except pymongo.errors.OperationFailure, e: + stories_db = MStory.objects(story_hash__in=unread_story_hashes)[:25] + stories = Feed.format_stories(stories_db, self.feed_id) unread_stories = [] for story in stories: diff --git a/clients/android/NewsBlur/build.gradle b/clients/android/NewsBlur/build.gradle index e2271c2d4..b48d5402c 100644 --- a/clients/android/NewsBlur/build.gradle +++ b/clients/android/NewsBlur/build.gradle @@ -9,7 +9,7 @@ buildscript { google() } dependencies { - classpath 'com.android.tools.build:gradle:4.0.2' + classpath 'com.android.tools.build:gradle:4.1.0' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" } } @@ -29,15 +29,15 @@ apply plugin: 'checkstyle' dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" - implementation 'androidx.fragment:fragment:1.2.5' + implementation 'androidx.fragment:fragment-ktx:1.2.5' implementation 'androidx.recyclerview:recyclerview:1.1.0' implementation 'androidx.swiperefreshlayout:swiperefreshlayout:1.1.0' - implementation 'com.squareup.okhttp3:okhttp:3.12.12' + implementation 'com.squareup.okhttp3:okhttp:4.9.0' implementation 'com.google.code.gson:gson:2.8.6' - implementation 'com.android.billingclient:billing:3.0.0' + implementation 'com.android.billingclient:billing:3.0.1' implementation 'nl.dionsegijn:konfetti:1.2.2' implementation 'com.github.jinatonic.confetti:confetti:1.1.2' - implementation 'com.google.android.play:core:1.8.2' + implementation 'com.google.android.play:core:1.8.3' } android { @@ -46,8 +46,8 @@ android { applicationId "com.newsblur" minSdkVersion 21 targetSdkVersion 29 - versionCode 176 - versionName "10.1" + versionCode 177 + versionName "10.1.1" } compileOptions.with { sourceCompatibility = JavaVersion.VERSION_1_8 diff --git a/clients/android/NewsBlur/res/values/strings.xml b/clients/android/NewsBlur/res/values/strings.xml index 7fe76ade2..e3eccc599 100644 --- a/clients/android/NewsBlur/res/values/strings.xml +++ b/clients/android/NewsBlur/res/values/strings.xml @@ -447,6 +447,7 @@ Catching up reading actions... On its way... Cleaning up... + Sync saved stories actions… Fetching fresh stories... Storing%sunread stories... Storing text for %s stories... diff --git a/clients/android/NewsBlur/src/com/newsblur/database/BlurDatabaseHelper.java b/clients/android/NewsBlur/src/com/newsblur/database/BlurDatabaseHelper.java index af7d6ec14..487e42ba7 100644 --- a/clients/android/NewsBlur/src/com/newsblur/database/BlurDatabaseHelper.java +++ b/clients/android/NewsBlur/src/com/newsblur/database/BlurDatabaseHelper.java @@ -284,6 +284,19 @@ public class BlurDatabaseHelper { return hashes; } + public Set getStarredStoryHashes() { + String q = "SELECT " + DatabaseConstants.STORY_HASH + + " FROM " + DatabaseConstants.STORY_TABLE + + " WHERE " + DatabaseConstants.STORY_STARRED + " = 1" ; + Cursor c = dbRO.rawQuery(q, null); + Set hashes = new HashSet<>(c.getCount()); + while (c.moveToNext()) { + hashes.add(c.getString(c.getColumnIndexOrThrow(DatabaseConstants.STORY_HASH))); + } + c.close(); + return hashes; + } + public Set getAllStoryImages() { Cursor c = dbRO.query(DatabaseConstants.STORY_TABLE, new String[]{DatabaseConstants.STORY_IMAGE_URLS}, null, null, null, null, null); Set urls = new HashSet(c.getCount()); @@ -584,6 +597,22 @@ public class BlurDatabaseHelper { } } + public void markStoryHashesStarred(Collection hashes, boolean isStarred) { + synchronized (RW_MUTEX) { + dbRW.beginTransaction(); + try { + ContentValues values = new ContentValues(); + values.put(DatabaseConstants.STORY_STARRED, isStarred); + for (String hash : hashes) { + dbRW.update(DatabaseConstants.STORY_TABLE, values, DatabaseConstants.STORY_HASH + " = ?", new String[]{hash}); + } + dbRW.setTransactionSuccessful(); + } finally { + dbRW.endTransaction(); + } + } + } + public void setFeedsActive(Set feedIds, boolean active) { synchronized (RW_MUTEX) { dbRW.beginTransaction(); diff --git a/clients/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java b/clients/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java index c11b40778..590eee923 100644 --- a/clients/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java +++ b/clients/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java @@ -587,8 +587,8 @@ public class FolderListFragment extends NbFragment implements OnCreateContextMen private void checkAccountFeedsLimit() { new Handler().postDelayed(() -> { - if (adapter.totalActiveFeedCount > AppConstants.FREE_ACCOUNT_SITE_LIMIT && !PrefsUtils.getIsPremium(requireContext())) { - Intent intent = new Intent(requireActivity(), MuteConfig.class); + if (getActivity() != null && adapter.totalActiveFeedCount > AppConstants.FREE_ACCOUNT_SITE_LIMIT && !PrefsUtils.getIsPremium(getActivity())) { + Intent intent = new Intent(getActivity(), MuteConfig.class); startActivity(intent); } }, 2000); diff --git a/clients/android/NewsBlur/src/com/newsblur/network/APIConstants.java b/clients/android/NewsBlur/src/com/newsblur/network/APIConstants.java index d8bcd0320..a958d10f5 100644 --- a/clients/android/NewsBlur/src/com/newsblur/network/APIConstants.java +++ b/clients/android/NewsBlur/src/com/newsblur/network/APIConstants.java @@ -52,6 +52,7 @@ public class APIConstants { public static final String PATH_MARK_STORY_AS_UNREAD = "/reader/mark_story_as_unread/"; public static final String PATH_MARK_STORY_HASH_UNREAD = "/reader/mark_story_hash_as_unread/"; public static final String PATH_STARRED_STORIES = "/reader/starred_stories"; + public static final String PATH_STARRED_STORY_HASHES = "/reader/starred_story_hashes"; public static final String PATH_FEED_AUTOCOMPLETE = "/rss_feeds/feed_autocomplete"; public static final String PATH_LIKE_COMMENT = "/social/like_comment"; public static final String PATH_UNLIKE_COMMENT = "/social/remove_like_comment"; diff --git a/clients/android/NewsBlur/src/com/newsblur/network/APIManager.java b/clients/android/NewsBlur/src/com/newsblur/network/APIManager.java index dc1046037..cb92b9f3e 100644 --- a/clients/android/NewsBlur/src/com/newsblur/network/APIManager.java +++ b/clients/android/NewsBlur/src/com/newsblur/network/APIManager.java @@ -37,6 +37,7 @@ import com.newsblur.network.domain.LoginResponse; import com.newsblur.network.domain.NewsBlurResponse; import com.newsblur.network.domain.ProfileResponse; import com.newsblur.network.domain.RegisterResponse; +import com.newsblur.network.domain.StarredStoryHashesResponse; import com.newsblur.network.domain.StoriesResponse; import com.newsblur.network.domain.StoryTextResponse; import com.newsblur.network.domain.UnreadCountResponse; @@ -279,6 +280,11 @@ public class APIManager { return (UnreadStoryHashesResponse) response.getResponse(gson, UnreadStoryHashesResponse.class); } + public StarredStoryHashesResponse getStarredStoryHashes() { + APIResponse response = get(buildUrl(APIConstants.PATH_STARRED_STORY_HASHES)); + return response.getResponse(gson, StarredStoryHashesResponse.class); + } + public StoriesResponse getStoriesByHash(List storyHashes) { ValueMultimap values = new ValueMultimap(); for (String hash : storyHashes) { diff --git a/clients/android/NewsBlur/src/com/newsblur/network/domain/StarredStoryHashesResponse.kt b/clients/android/NewsBlur/src/com/newsblur/network/domain/StarredStoryHashesResponse.kt new file mode 100644 index 000000000..215c63097 --- /dev/null +++ b/clients/android/NewsBlur/src/com/newsblur/network/domain/StarredStoryHashesResponse.kt @@ -0,0 +1,7 @@ +package com.newsblur.network.domain + +import com.google.gson.annotations.SerializedName + +data class StarredStoryHashesResponse( + @SerializedName("starred_story_hashes") + val starredStoryHashes: Set = HashSet()) : NewsBlurResponse() \ No newline at end of file diff --git a/clients/android/NewsBlur/src/com/newsblur/serialization/StoryTypeAdapter.java b/clients/android/NewsBlur/src/com/newsblur/serialization/StoryTypeAdapter.java index 3a6f55e41..6f8dd6f77 100644 --- a/clients/android/NewsBlur/src/com/newsblur/serialization/StoryTypeAdapter.java +++ b/clients/android/NewsBlur/src/com/newsblur/serialization/StoryTypeAdapter.java @@ -24,7 +24,7 @@ public class StoryTypeAdapter implements JsonDeserializer { // any characters we don't want in the short description, such as newlines or placeholders private final static Pattern ShortContentExcludes = Pattern.compile("[\\uFFFC\\u000A\\u000B\\u000C\\u000D]"); - private final static Pattern httpSniff = Pattern.compile("(?:http):\\/\\/"); + private final static Pattern httpSniff = Pattern.compile("(?:http):\\//"); public StoryTypeAdapter() { this.gson = new GsonBuilder() @@ -43,9 +43,11 @@ public class StoryTypeAdapter implements JsonDeserializer { // replace http image urls with https if (httpSniff.matcher(story.content).find() && story.secureImageUrls != null && story.secureImageUrls.size() > 0) { - for (String httpUrl : story.secureImageUrls.keySet()) { - String httpsUrl = story.secureImageUrls.get(httpUrl); - story.content = story.content.replace(httpUrl, httpsUrl); + for (String url : story.secureImageUrls.keySet()) { + if (httpSniff.matcher(url).find()) { + String httpsUrl = story.secureImageUrls.get(url); + story.content = story.content.replace(url, httpsUrl); + } } } diff --git a/clients/android/NewsBlur/src/com/newsblur/service/NBSyncService.java b/clients/android/NewsBlur/src/com/newsblur/service/NBSyncService.java index 22f4b395d..a0b5bdc29 100644 --- a/clients/android/NewsBlur/src/com/newsblur/service/NBSyncService.java +++ b/clients/android/NewsBlur/src/com/newsblur/service/NBSyncService.java @@ -133,6 +133,7 @@ public class NBSyncService extends JobService { private List outstandingStartParams = new ArrayList(); private boolean mainSyncRunning = false; CleanupService cleanupService; + StarredService starredService; OriginalTextService originalTextService; UnreadsService unreadsService; ImagePrefetchService imagePrefetchService; @@ -166,6 +167,7 @@ public class NBSyncService extends JobService { dbHelper = new BlurDatabaseHelper(this); iconCache = FileCache.asIconCache(this); cleanupService = new CleanupService(this); + starredService = new StarredService(this); originalTextService = new OriginalTextService(this); unreadsService = new UnreadsService(this); imagePrefetchService = new ImagePrefetchService(this); @@ -612,6 +614,7 @@ public class NBSyncService extends JobService { UnreadsService.doMetadata(); unreadsService.start(); cleanupService.start(); + starredService.start(); } finally { FFSyncRunning = false; @@ -952,6 +955,7 @@ public class NBSyncService extends JobService { //Log.d(this, "checking completion"); if (mainSyncRunning) return; if ((cleanupService != null) && cleanupService.isRunning()) return; + if ((starredService != null) && starredService.isRunning()) return; if ((originalTextService != null) && originalTextService.isRunning()) return; if ((unreadsService != null) && unreadsService.isRunning()) return; if ((imagePrefetchService != null) && imagePrefetchService.isRunning()) return; @@ -1036,6 +1040,7 @@ public class NBSyncService extends JobService { if (HousekeepingRunning) return context.getResources().getString(R.string.sync_status_housekeeping); if (FFSyncRunning) return context.getResources().getString(R.string.sync_status_ffsync); if (CleanupService.activelyRunning) return context.getResources().getString(R.string.sync_status_cleanup); + if (StarredService.activelyRunning) return context.getResources().getString(R.string.sync_status_starred); if (brief && !AppConstants.VERBOSE_LOG) return null; if (ActionsRunning) return String.format(context.getResources().getString(R.string.sync_status_actions), lastActionCount); if (RecountsRunning) return context.getResources().getString(R.string.sync_status_recounts); @@ -1196,6 +1201,7 @@ public class NBSyncService extends JobService { } if (cleanupService != null) cleanupService.shutdown(); if (unreadsService != null) unreadsService.shutdown(); + if (starredService != null) starredService.shutdown(); if (originalTextService != null) originalTextService.shutdown(); if (imagePrefetchService != null) imagePrefetchService.shutdown(); if (primaryExecutor != null) { diff --git a/clients/android/NewsBlur/src/com/newsblur/service/StarredService.kt b/clients/android/NewsBlur/src/com/newsblur/service/StarredService.kt new file mode 100644 index 000000000..c90423cd1 --- /dev/null +++ b/clients/android/NewsBlur/src/com/newsblur/service/StarredService.kt @@ -0,0 +1,37 @@ +package com.newsblur.service + +class StarredService(parent: NBSyncService) : SubService(parent) { + + companion object { + @JvmField + var activelyRunning = false + } + + override fun exec() { + activelyRunning = true + + if (parent.stopSync()) return + + // get all starred story hashes from remote db + val starredHashesResponse = parent.apiManager.starredStoryHashes + + if (parent.stopSync()) return + + // get all starred story hashes from local db + val localStoryHashes = parent.dbHelper.starredStoryHashes + + if (parent.stopSync()) return + + val newStarredHashes = starredHashesResponse.starredStoryHashes.minus(localStoryHashes) + val invalidStarredHashes = localStoryHashes.minus(starredHashesResponse.starredStoryHashes) + + if (newStarredHashes.isNotEmpty()) { + parent.dbHelper.markStoryHashesStarred(newStarredHashes, true) + } + if (invalidStarredHashes.isNotEmpty()) { + parent.dbHelper.markStoryHashesStarred(invalidStarredHashes, false) + } + + activelyRunning = false + } +} \ No newline at end of file diff --git a/clients/android/NewsBlur/src/com/newsblur/util/PrefsUtils.java b/clients/android/NewsBlur/src/com/newsblur/util/PrefsUtils.java index c19d262fb..5aa221045 100644 --- a/clients/android/NewsBlur/src/com/newsblur/util/PrefsUtils.java +++ b/clients/android/NewsBlur/src/com/newsblur/util/PrefsUtils.java @@ -792,7 +792,7 @@ public class PrefsUtils { public static ThemeValue getSelectedTheme(Context context) { SharedPreferences prefs = context.getSharedPreferences(PrefConstants.PREFERENCES, 0); - String value = prefs.getString(PrefConstants.THEME, ThemeValue.LIGHT.name()); + String value = prefs.getString(PrefConstants.THEME, ThemeValue.AUTO.name()); // check for legacy hard-coded values. this can go away once installs of v152 or earlier are minimized if (value.equals("light")) { setSelectedTheme(context, ThemeValue.LIGHT); diff --git a/utils/story_functions.py b/utils/story_functions.py index 4f0a3d3a1..a218af45f 100644 --- a/utils/story_functions.py +++ b/utils/story_functions.py @@ -191,6 +191,9 @@ def pre_process_story(entry, encoding): entry['title'] = strip_tags(entry.get('title')) entry['author'] = strip_tags(entry.get('author')) + if not entry['author']: + entry['author'] = strip_tags(entry.get('credit')) + entry['story_content'] = attach_media_scripts(entry['story_content'])