public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-09-18 21:43:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'django1.6' into django1.7

Browse source 
* django1.6:
  Fixing DB Monitor.
  Updating User Agent for all fetchers.
  Ignoring VSCode.
  Checking the 'credit' field if author not found. Works for TechRepublic, which I suspect is a one off.
  Handling multiple newsletter feeds.
  Attempting a fix of stories taking too long to sort while counting unreads.
  Fixing newsletter search.
  Fixing newsletter search.
  Newsletters feeds now de-duplicate based on sender email as well as sender name. This should cut down on duplicates.
  Android v10.1.1.
  Ignoring cache for homepage user.
  Gotta recreate empty_values somehow
  Fix exception
  Update dependencies
  #1380 Use image proxy only for http images
  #1379 Automatic light/dark theme
  #1367 Starred stories service
This commit is contained in:
Samuel Clay 2020-11-30 18:01:42 -05:00
commit 8ee277c7d3
23 changed files with 154 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -11,6 +11,7 @@ media/iphone/build
build/
.DS_Store
**/*.perspectivev*
.vscode/*
data/
config/certificates
**/*.xcuserstate

21
apps/newsletters/models.py
View file

@ -31,9 +31,26 @@ class EmailNewsletter:
return
usf.add_folder('', 'Newsletters')
# First look for the email address
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.MultipleObjectsReturned:
feeds = Feed.objects.filter(feed_address=feed_address).limit(1)
if feeds.count():
feed = feeds[0]
except Feed.DoesNotExist:
feed = None
# If not found, check among titles user has subscribed to
if not feed:
newsletter_subs = UserSubscription.objects.filter(user=user, feed__feed_address__contains="newsletter:").only('feed')
newsletter_feed_ids = [us.feed.pk for us in newsletter_subs]
feeds = Feed.objects.filter(feed_title__iexact=sender_name, pk__in=newsletter_feed_ids)
if feeds.count():
feed = feeds[0]
# Create a new feed if it doesn't exist by sender name or email
if not feed:
feed = Feed.objects.create(feed_address=feed_address,
feed_link='http://' + sender_domain,
feed_title=sender_name,
@ -148,8 +165,8 @@ class EmailNewsletter:
return profile.user
def _feed_address(self, user, sender):
return 'newsletter:%s:%s' % (user.pk, sender)
def _feed_address(self, user, sender_email):
return 'newsletter:%s:%s' % (user.pk, sender_email)
def _split_sender(self, sender):
tokens = re.search('(.*?) <(.*?)@(.*?)>', sender)

3
apps/reader/models.py
View file

@ -769,6 +769,9 @@ class UserSubscription(models.Model):
except pymongo.errors.OperationFailure, e:
stories_db = MStory.objects(story_hash__in=unread_story_hashes)[:100]
stories = Feed.format_stories(stories_db, self.feed_id)
except pymongo.errors.OperationFailure, e:
stories_db = MStory.objects(story_hash__in=unread_story_hashes)[:25]
stories = Feed.format_stories(stories_db, self.feed_id)
unread_stories = []
for story in stories:

8
apps/rss_feeds/icon_importer.py
View file

@ -275,14 +275,12 @@ class IconImporter(object):
@timelimit(30)
def _1(url):
headers = {
'User-Agent': 'NewsBlur Favicon Fetcher - %s subscriber%s - %s '
'(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
'Safari/534.48.3)' %
'User-Agent': 'NewsBlur Favicon Fetcher - %s subscriber%s - %s %s' %
(
self.feed.num_subscribers,
's' if self.feed.num_subscribers != 1 else '',
self.feed.permalink
self.feed.permalink,
self.feed.fake_user_agent,
),
'Connection': 'close',
'Accept': 'image/png,image/x-icon,image/*;q=0.9,*/*;q=0.8'

10
apps/rss_feeds/models.py
View file

@ -1117,20 +1117,20 @@ class Feed(models.Model):
# A known workaround is using facebook's user agent.
return 'facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)'
ua = ('NewsBlur Feed Fetcher - %s subscriber%s - %s '
'(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/56.0.2924.87 Safari/537.36)' % (
ua = ('NewsBlur Feed Fetcher - %s subscriber%s - %s %s' % (
self.num_subscribers,
's' if self.num_subscribers != 1 else '',
self.permalink,
self.fake_user_agent,
))
return ua
@property
def fake_user_agent(self):
ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
ua = ('("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
'AppleWebKit/605.1.15 (KHTML, like Gecko) '
'Version/14.0.1 Safari/605.1.15")')
return ua

6
apps/rss_feeds/page_importer.py
View file

@ -51,13 +51,11 @@ class PageImporter(object):
@property
def headers(self):
return {
'User-Agent': 'NewsBlur Page Fetcher - %s subscriber%s - %s '
'(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
'Safari/534.48.3)' % (
'User-Agent': 'NewsBlur Page Fetcher - %s subscriber%s - %s %s' % (
self.feed.num_subscribers,
's' if self.feed.num_subscribers != 1 else '',
self.feed.permalink,
self.feed.fake_user_agent,
),
}

8
apps/rss_feeds/text_importer.py
View file

@ -37,13 +37,11 @@ class TextImporter:
def headers(self):
num_subscribers = getattr(self.feed, 'num_subscribers', 0)
return {
'User-Agent': 'NewsBlur Content Fetcher - %s subscriber%s - %s '
'(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
'Safari/534.48.3)' % (
'User-Agent': 'NewsBlur Content Fetcher - %s subscriber%s - %s %s' % (
num_subscribers,
's' if num_subscribers != 1 else '',
getattr(self.feed, 'permalink', '')
getattr(self.feed, 'permalink', ''),
self.feed.fake_user_agent,
),
}

5
apps/social/models.py
View file

@ -2312,10 +2312,7 @@ class MSharedStory(mongo.DynamicDocument):
return self.image_sizes
headers = {
'User-Agent': 'NewsBlur Image Fetcher - %s '
'(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
'Safari/534.48.3)' % (
'User-Agent': 'NewsBlur Image Fetcher - %s' % (
settings.NEWSBLUR_URL
),
}

14
clients/android/NewsBlur/build.gradle
View file

@ -9,7 +9,7 @@ buildscript {
google()
}
dependencies {
classpath 'com.android.tools.build:gradle:4.0.2'
classpath 'com.android.tools.build:gradle:4.1.0'
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
}
}
@ -29,15 +29,15 @@ apply plugin: 'checkstyle'
dependencies {
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
implementation 'androidx.fragment:fragment:1.2.5'
implementation 'androidx.fragment:fragment-ktx:1.2.5'
implementation 'androidx.recyclerview:recyclerview:1.1.0'
implementation 'androidx.swiperefreshlayout:swiperefreshlayout:1.1.0'
implementation 'com.squareup.okhttp3:okhttp:3.12.12'
implementation 'com.squareup.okhttp3:okhttp:4.9.0'
implementation 'com.google.code.gson:gson:2.8.6'
implementation 'com.android.billingclient:billing:3.0.0'
implementation 'com.android.billingclient:billing:3.0.1'
implementation 'nl.dionsegijn:konfetti:1.2.2'
implementation 'com.github.jinatonic.confetti:confetti:1.1.2'
implementation 'com.google.android.play:core:1.8.2'
implementation 'com.google.android.play:core:1.8.3'
}
android {
@ -46,8 +46,8 @@ android {
applicationId "com.newsblur"
minSdkVersion 21
targetSdkVersion 29
versionCode 176
versionName "10.1"
versionCode 177
versionName "10.1.1"
}
compileOptions.with {
sourceCompatibility = JavaVersion.VERSION_1_8

1
clients/android/NewsBlur/res/values/strings.xml
View file

@ -447,6 +447,7 @@
<string name="sync_status_recounts">Catching up reading actions...</string>
<string name="sync_status_ffsync">On its way...</string>
<string name="sync_status_cleanup">Cleaning up...</string>
<string name="sync_status_starred">Sync saved stories actions…</string>
<string name="sync_status_stories">Fetching fresh stories...</string>
<string name="sync_status_unreads">Storing%sunread stories...</string>
<string name="sync_status_text">Storing text for %s stories...</string>

29
clients/android/NewsBlur/src/com/newsblur/database/BlurDatabaseHelper.java
View file

@ -284,6 +284,19 @@ public class BlurDatabaseHelper {
return hashes;
}
public Set<String> getStarredStoryHashes() {
String q = "SELECT " + DatabaseConstants.STORY_HASH +
" FROM " + DatabaseConstants.STORY_TABLE +
" WHERE " + DatabaseConstants.STORY_STARRED + " = 1" ;
Cursor c = dbRO.rawQuery(q, null);
Set<String> hashes = new HashSet<>(c.getCount());
while (c.moveToNext()) {
hashes.add(c.getString(c.getColumnIndexOrThrow(DatabaseConstants.STORY_HASH)));
}
c.close();
return hashes;
}
public Set<String> getAllStoryImages() {
Cursor c = dbRO.query(DatabaseConstants.STORY_TABLE, new String[]{DatabaseConstants.STORY_IMAGE_URLS}, null, null, null, null, null);
Set<String> urls = new HashSet<String>(c.getCount());
@ -584,6 +597,22 @@ public class BlurDatabaseHelper {
}
}
public void markStoryHashesStarred(Collection<String> hashes, boolean isStarred) {
synchronized (RW_MUTEX) {
dbRW.beginTransaction();
try {
ContentValues values = new ContentValues();
values.put(DatabaseConstants.STORY_STARRED, isStarred);
for (String hash : hashes) {
dbRW.update(DatabaseConstants.STORY_TABLE, values, DatabaseConstants.STORY_HASH + " = ?", new String[]{hash});
}
dbRW.setTransactionSuccessful();
} finally {
dbRW.endTransaction();
}
}
}
public void setFeedsActive(Set<String> feedIds, boolean active) {
synchronized (RW_MUTEX) {
dbRW.beginTransaction();

4
clients/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java
View file

@ -587,8 +587,8 @@ public class FolderListFragment extends NbFragment implements OnCreateContextMen
private void checkAccountFeedsLimit() {
new Handler().postDelayed(() -> {
if (adapter.totalActiveFeedCount > AppConstants.FREE_ACCOUNT_SITE_LIMIT && !PrefsUtils.getIsPremium(requireContext())) {
Intent intent = new Intent(requireActivity(), MuteConfig.class);
if (getActivity() != null && adapter.totalActiveFeedCount > AppConstants.FREE_ACCOUNT_SITE_LIMIT && !PrefsUtils.getIsPremium(getActivity())) {
Intent intent = new Intent(getActivity(), MuteConfig.class);
startActivity(intent);
}
}, 2000);

1
clients/android/NewsBlur/src/com/newsblur/network/APIConstants.java
View file

@ -52,6 +52,7 @@ public class APIConstants {
public static final String PATH_MARK_STORY_AS_UNREAD = "/reader/mark_story_as_unread/";
public static final String PATH_MARK_STORY_HASH_UNREAD = "/reader/mark_story_hash_as_unread/";
public static final String PATH_STARRED_STORIES = "/reader/starred_stories";
public static final String PATH_STARRED_STORY_HASHES = "/reader/starred_story_hashes";
public static final String PATH_FEED_AUTOCOMPLETE = "/rss_feeds/feed_autocomplete";
public static final String PATH_LIKE_COMMENT = "/social/like_comment";
public static final String PATH_UNLIKE_COMMENT = "/social/remove_like_comment";

6
clients/android/NewsBlur/src/com/newsblur/network/APIManager.java
View file

@ -37,6 +37,7 @@ import com.newsblur.network.domain.LoginResponse;
import com.newsblur.network.domain.NewsBlurResponse;
import com.newsblur.network.domain.ProfileResponse;
import com.newsblur.network.domain.RegisterResponse;
import com.newsblur.network.domain.StarredStoryHashesResponse;
import com.newsblur.network.domain.StoriesResponse;
import com.newsblur.network.domain.StoryTextResponse;
import com.newsblur.network.domain.UnreadCountResponse;
@ -279,6 +280,11 @@ public class APIManager {
return (UnreadStoryHashesResponse) response.getResponse(gson, UnreadStoryHashesResponse.class);
}
public StarredStoryHashesResponse getStarredStoryHashes() {
APIResponse response = get(buildUrl(APIConstants.PATH_STARRED_STORY_HASHES));
return response.getResponse(gson, StarredStoryHashesResponse.class);
}
public StoriesResponse getStoriesByHash(List<String> storyHashes) {
ValueMultimap values = new ValueMultimap();
for (String hash : storyHashes) {

7
clients/android/NewsBlur/src/com/newsblur/network/domain/StarredStoryHashesResponse.kt Normal file
View file

@ -0,0 +1,7 @@
package com.newsblur.network.domain
import com.google.gson.annotations.SerializedName
data class StarredStoryHashesResponse(
@SerializedName("starred_story_hashes")
val starredStoryHashes: Set<String> = HashSet()) : NewsBlurResponse()

10
clients/android/NewsBlur/src/com/newsblur/serialization/StoryTypeAdapter.java
View file

@ -24,7 +24,7 @@ public class StoryTypeAdapter implements JsonDeserializer<Story> {
// any characters we don't want in the short description, such as newlines or placeholders
private final static Pattern ShortContentExcludes = Pattern.compile("[\\uFFFC\\u000A\\u000B\\u000C\\u000D]");
private final static Pattern httpSniff = Pattern.compile("(?:http):\\/\\/");
private final static Pattern httpSniff = Pattern.compile("(?:http):\\//");
public StoryTypeAdapter() {
this.gson = new GsonBuilder()
@ -43,9 +43,11 @@ public class StoryTypeAdapter implements JsonDeserializer<Story> {
// replace http image urls with https
if (httpSniff.matcher(story.content).find() && story.secureImageUrls != null && story.secureImageUrls.size() > 0) {
for (String httpUrl : story.secureImageUrls.keySet()) {
String httpsUrl = story.secureImageUrls.get(httpUrl);
story.content = story.content.replace(httpUrl, httpsUrl);
for (String url : story.secureImageUrls.keySet()) {
if (httpSniff.matcher(url).find()) {
String httpsUrl = story.secureImageUrls.get(url);
story.content = story.content.replace(url, httpsUrl);
}
}
}

6
clients/android/NewsBlur/src/com/newsblur/service/NBSyncService.java
View file

@ -133,6 +133,7 @@ public class NBSyncService extends JobService {
private List<JobParameters> outstandingStartParams = new ArrayList<JobParameters>();
private boolean mainSyncRunning = false;
CleanupService cleanupService;
StarredService starredService;
OriginalTextService originalTextService;
UnreadsService unreadsService;
ImagePrefetchService imagePrefetchService;
@ -166,6 +167,7 @@ public class NBSyncService extends JobService {
dbHelper = new BlurDatabaseHelper(this);
iconCache = FileCache.asIconCache(this);
cleanupService = new CleanupService(this);
starredService = new StarredService(this);
originalTextService = new OriginalTextService(this);
unreadsService = new UnreadsService(this);
imagePrefetchService = new ImagePrefetchService(this);
@ -612,6 +614,7 @@ public class NBSyncService extends JobService {
UnreadsService.doMetadata();
unreadsService.start();
cleanupService.start();
starredService.start();
} finally {
FFSyncRunning = false;
@ -952,6 +955,7 @@ public class NBSyncService extends JobService {
//Log.d(this, "checking completion");
if (mainSyncRunning) return;
if ((cleanupService != null) && cleanupService.isRunning()) return;
if ((starredService != null) && starredService.isRunning()) return;
if ((originalTextService != null) && originalTextService.isRunning()) return;
if ((unreadsService != null) && unreadsService.isRunning()) return;
if ((imagePrefetchService != null) && imagePrefetchService.isRunning()) return;
@ -1036,6 +1040,7 @@ public class NBSyncService extends JobService {
if (HousekeepingRunning) return context.getResources().getString(R.string.sync_status_housekeeping);
if (FFSyncRunning) return context.getResources().getString(R.string.sync_status_ffsync);
if (CleanupService.activelyRunning) return context.getResources().getString(R.string.sync_status_cleanup);
if (StarredService.activelyRunning) return context.getResources().getString(R.string.sync_status_starred);
if (brief && !AppConstants.VERBOSE_LOG) return null;
if (ActionsRunning) return String.format(context.getResources().getString(R.string.sync_status_actions), lastActionCount);
if (RecountsRunning) return context.getResources().getString(R.string.sync_status_recounts);
@ -1196,6 +1201,7 @@ public class NBSyncService extends JobService {
}
if (cleanupService != null) cleanupService.shutdown();
if (unreadsService != null) unreadsService.shutdown();
if (starredService != null) starredService.shutdown();
if (originalTextService != null) originalTextService.shutdown();
if (imagePrefetchService != null) imagePrefetchService.shutdown();
if (primaryExecutor != null) {

37
clients/android/NewsBlur/src/com/newsblur/service/StarredService.kt Normal file
View file

@ -0,0 +1,37 @@
package com.newsblur.service
class StarredService(parent: NBSyncService) : SubService(parent) {
companion object {
@JvmField
var activelyRunning = false
}
override fun exec() {
activelyRunning = true
if (parent.stopSync()) return
// get all starred story hashes from remote db
val starredHashesResponse = parent.apiManager.starredStoryHashes
if (parent.stopSync()) return
// get all starred story hashes from local db
val localStoryHashes = parent.dbHelper.starredStoryHashes
if (parent.stopSync()) return
val newStarredHashes = starredHashesResponse.starredStoryHashes.minus(localStoryHashes)
val invalidStarredHashes = localStoryHashes.minus(starredHashesResponse.starredStoryHashes)
if (newStarredHashes.isNotEmpty()) {
parent.dbHelper.markStoryHashesStarred(newStarredHashes, true)
}
if (invalidStarredHashes.isNotEmpty()) {
parent.dbHelper.markStoryHashesStarred(invalidStarredHashes, false)
}
activelyRunning = false
}
}

2
clients/android/NewsBlur/src/com/newsblur/util/PrefsUtils.java
View file

@ -792,7 +792,7 @@ public class PrefsUtils {
public static ThemeValue getSelectedTheme(Context context) {
SharedPreferences prefs = context.getSharedPreferences(PrefConstants.PREFERENCES, 0);
String value = prefs.getString(PrefConstants.THEME, ThemeValue.LIGHT.name());
String value = prefs.getString(PrefConstants.THEME, ThemeValue.AUTO.name());
// check for legacy hard-coded values. this can go away once installs of v152 or earlier are minimized
if (value.equals("light")) {
setSelectedTheme(context, ThemeValue.LIGHT);

2
config/haproxy.conf.template
View file

@ -127,7 +127,7 @@ backend postgres
backend mongo
option httpchk GET /db_check/mongo
server mongo-db20 db20:5000 check inter 2000ms
server mongo-db20e db20e:5000 check inter 2000ms
server mongo-db23a db23a:5000 check inter 2000ms
server mongo-db25a db25a:5000 check inter 2000ms
server mongo-db30 db30:5000 check inter 2000ms

5
flask/db_monitor.py
View file

@ -1,7 +1,10 @@
from flask import Flask, abort
import flask_settings as settings
import psycopg2
import MySQLdb
try:
import MySQLdb
except ImportError:
pass
import pymongo
import redis
import pyes

3
utils/story_functions.py
View file

@ -191,6 +191,9 @@ def pre_process_story(entry, encoding):
entry['title'] = strip_tags(entry.get('title'))
entry['author'] = strip_tags(entry.get('author'))
if not entry['author']:
entry['author'] = strip_tags(entry.get('credit'))
entry['story_content'] = attach_media_scripts(entry['story_content'])

15
utils/user_functions.py
View file

@ -90,15 +90,12 @@ def get_user(request):
pass
if user.is_anonymous():
user = cache.get('user:%s' % settings.HOMEPAGE_USERNAME, None)
if not user:
try:
user = User.objects.get(username=settings.HOMEPAGE_USERNAME)
cache.set('user:%s' % user, user)
except User.DoesNotExist:
user = User.objects.create(username=settings.HOMEPAGE_USERNAME)
user.set_password('')
user.save()
try:
user = User.objects.get(username=settings.HOMEPAGE_USERNAME)
except User.DoesNotExist:
user = User.objects.create(username=settings.HOMEPAGE_USERNAME)
user.set_password('')
user.save()
return user
def invalidate_template_cache(fragment_name, *variables):