diff --git a/apps/reader/models.py b/apps/reader/models.py index 90e19d727..79bbcc036 100644 --- a/apps/reader/models.py +++ b/apps/reader/models.py @@ -96,13 +96,17 @@ class UserSubscription(models.Model): def sync_redis(self, skip_feed=False): r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) + UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD+1) - if not skip_feed: - self.feed.sync_redis() + userstories = MUserStory.objects.filter(feed_id=self.feed_id, user_id=self.user_id, + read_date__gte=UNREAD_CUTOFF) + total = userstories.count() + logging.debug(" ---> ~SN~FMSyncing ~SB%s~SN stories (%s)" % (total, self)) - userstories = MUserStory.objects.filter(feed_id=self.feed_id, user_id=self.user_id) + pipeline = r.pipeline() for userstory in userstories: - userstory.sync_redis(r=r) + userstory.sync_redis(pipeline=pipeline) + pipeline.execute() def get_stories(self, offset=0, limit=6, order='newest', read_filter='all', withscores=False): r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) @@ -660,10 +664,12 @@ class MUserStory(mongo.Document): return story.id - def sync_redis(self, r=None): - if not r: + def sync_redis(self, r=None, pipeline=None): + if pipeline: + r = pipeline + elif not r: r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) - + if self.story_db_id: all_read_stories_key = 'RS:%s' % (self.user_id) r.sadd(all_read_stories_key, self.story_db_id) @@ -682,9 +688,15 @@ class MUserStory(mongo.Document): @classmethod def sync_all_redis(cls, user_id=None, feed_id=None, force=False): r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) - UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD*2) - - if feed_id: + UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD+1) + + if feed_id and user_id: + read_stories = cls.objects.filter(user_id=user_id, + feed_id=feed_id, + read_date__gte=UNREAD_CUTOFF) + key = "RS:%s:%s" % (user_id, feed_id) + r.delete(key) + elif feed_id: read_stories = cls.objects.filter(feed_id=feed_id, read_date__gte=UNREAD_CUTOFF) keys = r.keys("RS:*:%s" % feed_id) print " ---> Deleting %s redis keys: %s" % (len(keys), keys) @@ -703,11 +715,18 @@ class MUserStory(mongo.Document): raise "Specify user_id, feed_id, or force." total = read_stories.count() - print " ---> Syncing %s stories (%s)" % (total, user_id or feed_id) + logging.debug(" ---> ~SN~FMSyncing ~SB%s~SN stories (%s/%s)" % (total, user_id, feed_id)) + pipeline = None for i, read_story in enumerate(read_stories): + if not pipeline: + pipeline = r.pipeline() if (i+1) % 1000 == 0: print " ---> %s/%s" % (i+1, total) - read_story.sync_redis(r) + pipeline.execute() + pipeline = r.pipeline() + read_story.sync_redis(r, pipeline=pipeline) + if pipeline: + pipeline.execute() class UserSubscriptionFolders(models.Model): """ diff --git a/apps/reader/views.py b/apps/reader/views.py index cb117f60f..31fbaf66a 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -3,6 +3,7 @@ import time import boto import redis import requests +import random from django.shortcuts import get_object_or_404 from django.shortcuts import render from django.contrib.auth.decorators import login_required @@ -494,6 +495,9 @@ def load_single_feed(request, feed_id): except UserSubscription.DoesNotExist: usersub = None + if usersub and random.random() < 0.01: + usersub.sync_redis() + if query: stories = feed.find_stories(query, offset=offset, limit=limit) elif usersub and (read_filter == 'unread' or order == 'oldest'): diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 856440ed1..5e80a975b 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -177,6 +177,8 @@ class Feed(models.Model): self.next_scheduled_update = datetime.datetime.utcnow() if not self.queued_date: self.queued_date = datetime.datetime.utcnow() + self.fix_google_alerts_urls() + feed_address = self.feed_address or "" feed_link = self.feed_link or "" self.hash_address_and_link = hashlib.sha1(feed_address+feed_link).hexdigest() @@ -228,18 +230,18 @@ class Feed(models.Model): return MStory.sync_all_redis(self.pk) def sync_autocompletion(self): - if self.num_subscribers <= 1: return + if self.num_subscribers <= 10: return if self.branch_from_feed: return if any(t in self.feed_address for t in ['token', 'private']): return engine = RedisEngine(prefix="FT", connection_pool=settings.REDIS_AUTOCOMPLETE_POOL) engine.store(self.pk, title=self.feed_title) - engine.boost(self.pk, self.num_subscribers) + engine.boost(self.pk, min(1, self.num_subscribers / 10000.)) parts = urlparse(self.feed_address) engine = RedisEngine(prefix="FA", connection_pool=settings.REDIS_AUTOCOMPLETE_POOL) engine.store(self.pk, title=parts.hostname) - engine.boost(self.pk, self.num_subscribers) + engine.boost(self.pk, min(1, self.num_subscribers / 10000.)) @classmethod def autocomplete(self, prefix, limit=5): @@ -269,6 +271,14 @@ class Feed(models.Model): def merge_feeds(cls, *args, **kwargs): return merge_feeds(*args, **kwargs) + def fix_google_alerts_urls(self): + if (self.feed_address.startswith('http://user/') and + '/state/com.google/alerts/' in self.feed_address): + match = re.match(r"http://user/(\d+)/state/com.google/alerts/(\d+)", self.feed_address) + if match: + user_id, alert_id = match.groups() + self.feed_address = "http://www.google.com/alerts/feeds/%s/%s" % (user_id, alert_id) + @classmethod def schedule_feed_fetches_immediately(cls, feed_ids): logging.info(" ---> ~SN~FMScheduling immediate fetch of ~SB%s~SN feeds..." % diff --git a/apps/rss_feeds/tasks.py b/apps/rss_feeds/tasks.py index 53be0b686..23a7d82cd 100644 --- a/apps/rss_feeds/tasks.py +++ b/apps/rss_feeds/tasks.py @@ -33,8 +33,8 @@ class TaskFeeds(Task): r.zcard('scheduled_updates'))) # Regular feeds - if tasked_feeds_size < 1000: - feeds = r.srandmember('queued_feeds', 1000) + if tasked_feeds_size < 2000: + feeds = r.srandmember('queued_feeds', 1500) Feed.task_feeds(feeds, verbose=True) active_count = len(feeds) else: diff --git a/media/android/NewsBlur/AndroidManifest.xml b/media/android/NewsBlur/AndroidManifest.xml index 893692046..6ba0ed9f5 100644 --- a/media/android/NewsBlur/AndroidManifest.xml +++ b/media/android/NewsBlur/AndroidManifest.xml @@ -36,6 +36,7 @@ NewsBlur @@ -147,4 +146,5 @@ %d COMMENTS All stories marked as read Unknown User + Delete feed \"%s\"? diff --git a/media/android/NewsBlur/src/com/newsblur/activity/FeedItemsList.java b/media/android/NewsBlur/src/com/newsblur/activity/FeedItemsList.java index cd8eac60c..11858630c 100644 --- a/media/android/NewsBlur/src/com/newsblur/activity/FeedItemsList.java +++ b/media/android/NewsBlur/src/com/newsblur/activity/FeedItemsList.java @@ -5,6 +5,7 @@ import android.content.Intent; import android.database.Cursor; import android.net.Uri; import android.os.Bundle; +import android.support.v4.app.DialogFragment; import android.support.v4.app.FragmentTransaction; import android.widget.Toast; @@ -15,6 +16,7 @@ import com.newsblur.R; import com.newsblur.database.DatabaseConstants; import com.newsblur.database.FeedProvider; import com.newsblur.domain.Feed; +import com.newsblur.fragment.DeleteFeedFragment; import com.newsblur.fragment.FeedItemListFragment; import com.newsblur.fragment.SyncUpdateFragment; import com.newsblur.network.APIManager; @@ -24,7 +26,11 @@ import com.newsblur.service.SyncService; public class FeedItemsList extends ItemsList { public static final String EXTRA_FEED = "feedId"; + public static final String EXTRA_FEED_TITLE = "feedTitle"; + public static final String EXTRA_FOLDER_NAME = "folderName"; private String feedId; + private String feedTitle; + private String folderName; private APIManager apiManager; private boolean stopLoading = false; @@ -33,7 +39,9 @@ public class FeedItemsList extends ItemsList { super.onCreate(bundle); apiManager = new APIManager(this); feedId = getIntent().getStringExtra(EXTRA_FEED); - + feedTitle = getIntent().getStringExtra(EXTRA_FEED_TITLE); + folderName = getIntent().getStringExtra(EXTRA_FOLDER_NAME); + final Uri feedUri = FeedProvider.FEEDS_URI.buildUpon().appendPath(feedId).build(); Cursor cursor = getContentResolver().query(feedUri, null, FeedProvider.getStorySelectionFromState(currentState), null, null); cursor.moveToFirst(); @@ -58,12 +66,8 @@ public class FeedItemsList extends ItemsList { } public void deleteFeed() { - setSupportProgressBarIndeterminateVisibility(true); - final Intent intent = new Intent(Intent.ACTION_SYNC, null, this, SyncService.class); - intent.putExtra(SyncService.EXTRA_STATUS_RECEIVER, syncFragment.receiver); - intent.putExtra(SyncService.SYNCSERVICE_TASK, SyncService.EXTRA_TASK_DELETE_FEED); - intent.putExtra(SyncService.EXTRA_TASK_FEED_ID, Long.parseLong(feedId)); - startService(intent); + DialogFragment deleteFeedFragment = DeleteFeedFragment.newInstance(Long.parseLong(feedId), feedTitle, folderName); + deleteFeedFragment.show(fragmentManager, "dialog"); } @Override diff --git a/media/android/NewsBlur/src/com/newsblur/activity/Main.java b/media/android/NewsBlur/src/com/newsblur/activity/Main.java index 520b692e6..5f4b7d1ef 100644 --- a/media/android/NewsBlur/src/com/newsblur/activity/Main.java +++ b/media/android/NewsBlur/src/com/newsblur/activity/Main.java @@ -115,15 +115,6 @@ public class Main extends NbFragmentActivity implements StateChangedListener, Sy return super.onOptionsItemSelected(item); } - public void deleteFeed(long id, String foldername) { - setSupportProgressBarIndeterminateVisibility(true); - final Intent intent = new Intent(Intent.ACTION_SYNC, null, this, SyncService.class); - intent.putExtra(SyncService.EXTRA_STATUS_RECEIVER, syncFragment.receiver); - intent.putExtra(SyncService.SYNCSERVICE_TASK, SyncService.EXTRA_TASK_DELETE_FEED); - intent.putExtra(SyncService.EXTRA_TASK_FEED_ID, id); - startService(intent); - } - @Override public void changedState(int state) { folderFeedList.changeState(state); diff --git a/media/android/NewsBlur/src/com/newsblur/activity/RegisterProgress.java b/media/android/NewsBlur/src/com/newsblur/activity/RegisterProgress.java index b6603e008..85d8e48dc 100644 --- a/media/android/NewsBlur/src/com/newsblur/activity/RegisterProgress.java +++ b/media/android/NewsBlur/src/com/newsblur/activity/RegisterProgress.java @@ -4,14 +4,19 @@ import android.os.Bundle; import android.support.v4.app.FragmentManager; import android.support.v4.app.FragmentTransaction; +import com.actionbarsherlock.app.SherlockFragmentActivity; import com.newsblur.R; import com.newsblur.fragment.RegisterProgressFragment; -public class RegisterProgress extends NbFragmentActivity { +/** + * Show progress screen while registering request is being processed. This + * Activity doesn't extend NbFragmentActivity because it is one of the few + * Activities that will be shown while the user is still logged out. + */ +public class RegisterProgress extends SherlockFragmentActivity { private FragmentManager fragmentManager; private String currentTag = "fragment"; - private String TAG = "RegisterProgressActivity"; @Override protected void onCreate(Bundle bundle) { diff --git a/media/android/NewsBlur/src/com/newsblur/fragment/DeleteFeedFragment.java b/media/android/NewsBlur/src/com/newsblur/fragment/DeleteFeedFragment.java new file mode 100644 index 000000000..5b19f99a8 --- /dev/null +++ b/media/android/NewsBlur/src/com/newsblur/fragment/DeleteFeedFragment.java @@ -0,0 +1,119 @@ +package com.newsblur.fragment; + +import com.newsblur.R; +import com.newsblur.activity.Main; +import com.newsblur.database.FeedProvider; +import com.newsblur.network.APIManager; + +import android.app.Activity; +import android.net.Uri; +import android.os.AsyncTask; +import android.os.Bundle; +import android.support.v4.app.DialogFragment; +import android.support.v4.app.FragmentManager; +import android.view.LayoutInflater; +import android.view.View; +import android.view.ViewGroup; +import android.view.View.OnClickListener; +import android.widget.Button; +import android.widget.TextView; +import android.widget.Toast; + +public class DeleteFeedFragment extends DialogFragment { + private static final String FEED_ID = "feed_url"; + private static final String FEED_NAME = "feed_name"; + private static final String FOLDER_NAME = "folder_name"; + + private APIManager apiManager; + + public static DeleteFeedFragment newInstance(final long feedId, final String feedName, final String folderName) { + DeleteFeedFragment frag = new DeleteFeedFragment(); + Bundle args = new Bundle(); + args.putLong(FEED_ID, feedId); + args.putString(FEED_NAME, feedName); + args.putString(FOLDER_NAME, folderName); + frag.setArguments(args); + return frag; + } + + private FragmentManager fragmentManager; + private SyncUpdateFragment syncFragment; + + @Override + public void onCreate(Bundle savedInstanceState) { + setStyle(DialogFragment.STYLE_NO_TITLE, R.style.dialog); + super.onCreate(savedInstanceState); + + fragmentManager = super.getFragmentManager(); + + syncFragment = (SyncUpdateFragment) fragmentManager.findFragmentByTag(SyncUpdateFragment.TAG); + if (syncFragment == null) { + syncFragment = new SyncUpdateFragment(); + } + } + + @Override + public View onCreateView(LayoutInflater inflater, ViewGroup container, + Bundle savedInstanceState) { + final String deleteFeedString = getResources().getString(R.string.delete_feed_message); + + apiManager = new APIManager(getActivity()); + View v = inflater.inflate(R.layout.fragment_confirm_dialog, null); + TextView messageView = (TextView) v.findViewById(R.id.dialog_message); + messageView.setText(String.format(deleteFeedString, getArguments().getString(FEED_NAME))); + + Button okayButton = (Button) v.findViewById(R.id.dialog_button_okay); + okayButton.setOnClickListener(new OnClickListener() { + public void onClick(final View v) { + v.setEnabled(false); + + new AsyncTask() { + @Override + protected Boolean doInBackground(Void... arg) { + long feedId = getArguments().getLong(FEED_ID); + String folderName = getArguments().getString(FOLDER_NAME); + if (apiManager.deleteFeed(feedId, folderName)) { + Uri feedUri = FeedProvider.FEEDS_URI.buildUpon().appendPath(Long.toString(feedId)).build(); + DeleteFeedFragment.this.getActivity().getContentResolver().delete(feedUri, null, null); + return true; + } + else { + return false; + } + } + + @Override + protected void onPostExecute(Boolean result) { + Activity activity = DeleteFeedFragment.this.getActivity(); + if (result) { + Toast.makeText(activity, "Deleted feed", Toast.LENGTH_SHORT).show(); + DeleteFeedFragment.this.dismiss(); + // if called from main view then refresh otherwise it was + // called from the feed view so finish + if (activity instanceof Main) { + ((Main)activity).updateAfterSync(); + } + else { + activity.finish(); + } + } else { + Toast.makeText(activity, getResources().getString(R.string.error_deleting_feed), Toast.LENGTH_LONG).show(); + DeleteFeedFragment.this.dismiss(); + } + }; + }.execute(); + + } + }); + + Button cancelButton = (Button) v.findViewById(R.id.dialog_button_cancel); + cancelButton.setOnClickListener(new OnClickListener() { + public void onClick(View v) { + DeleteFeedFragment.this.dismiss(); + } + }); + + return v; + } + +} diff --git a/media/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java b/media/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java index db6a39747..0f2aa235d 100644 --- a/media/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java +++ b/media/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java @@ -11,6 +11,7 @@ import android.content.SharedPreferences; import android.database.Cursor; import android.os.AsyncTask; import android.os.Bundle; +import android.support.v4.app.DialogFragment; import android.support.v4.app.Fragment; import android.util.Log; import android.view.ContextMenu; @@ -31,7 +32,6 @@ import com.newsblur.R; import com.newsblur.activity.AllStoriesItemsList; import com.newsblur.activity.FeedItemsList; import com.newsblur.activity.ItemsList; -import com.newsblur.activity.Main; import com.newsblur.activity.NewsBlurApplication; import com.newsblur.activity.SocialFeedItemsList; import com.newsblur.database.DatabaseConstants; @@ -183,8 +183,14 @@ public class FolderListFragment extends Fragment implements OnGroupClickListener }.execute(Long.toString(info.id)); return true; } else if (item.getItemId() == R.id.menu_delete_feed) { - Toast.makeText(getActivity(), "Deleted feed", Toast.LENGTH_SHORT).show(); - ((Main) getActivity()).deleteFeed(info.id, null); + int childPosition = ExpandableListView.getPackedPositionChild(info.packedPosition); + int groupPosition = ExpandableListView.getPackedPositionGroup(info.packedPosition); + Cursor childCursor = folderAdapter.getChild(groupPosition, childPosition); + String feedTitle = childCursor.getString(childCursor.getColumnIndex(DatabaseConstants.FEED_TITLE)); + Cursor folderCursor = ((MixedExpandableListAdapter) list.getExpandableListAdapter()).getGroup(groupPosition); + String folderName = folderCursor.getString(folderCursor.getColumnIndex(DatabaseConstants.FOLDER_NAME)); + DialogFragment deleteFeedFragment = DeleteFeedFragment.newInstance(info.id, feedTitle, folderName); + deleteFeedFragment.show(getFragmentManager(), "dialog"); return true; } else if (item.getItemId() == R.id.menu_mark_folder_as_read) { int groupPosition = ExpandableListView.getPackedPositionGroup(info.packedPosition); @@ -330,7 +336,12 @@ public class FolderListFragment extends Fragment implements OnGroupClickListener final Intent intent = new Intent(getActivity(), FeedItemsList.class); Cursor childCursor = folderAdapter.getChild(groupPosition, childPosition); String feedId = childCursor.getString(childCursor.getColumnIndex(DatabaseConstants.FEED_ID)); + String feedTitle = childCursor.getString(childCursor.getColumnIndex(DatabaseConstants.FEED_TITLE)); + final Cursor folderCursor = ((MixedExpandableListAdapter) list.getExpandableListAdapter()).getGroup(groupPosition); + String folderName = folderCursor.getString(folderCursor.getColumnIndex(DatabaseConstants.FOLDER_NAME)); intent.putExtra(FeedItemsList.EXTRA_FEED, feedId); + intent.putExtra(FeedItemsList.EXTRA_FEED_TITLE, feedTitle); + intent.putExtra(FeedItemsList.EXTRA_FOLDER_NAME, folderName); intent.putExtra(ItemsList.EXTRA_STATE, currentState); getActivity().startActivityForResult(intent, FEEDCHECK ); } diff --git a/media/android/NewsBlur/src/com/newsblur/network/APIClient.java b/media/android/NewsBlur/src/com/newsblur/network/APIClient.java index a97d2d943..1cdf1aa75 100644 --- a/media/android/NewsBlur/src/com/newsblur/network/APIClient.java +++ b/media/android/NewsBlur/src/com/newsblur/network/APIClient.java @@ -173,6 +173,7 @@ public class APIClient { } final PrintWriter printWriter = new PrintWriter(connection.getOutputStream()); + Log.d(this.getClass().getName(), "parameterString = " + parameterString); printWriter.print(parameterString); printWriter.close(); diff --git a/media/android/NewsBlur/src/com/newsblur/network/APIManager.java b/media/android/NewsBlur/src/com/newsblur/network/APIManager.java index d46e31fbc..c936e4698 100644 --- a/media/android/NewsBlur/src/com/newsblur/network/APIManager.java +++ b/media/android/NewsBlur/src/com/newsblur/network/APIManager.java @@ -700,12 +700,15 @@ public class APIManager { // TODO find a better way to identify these failed responses boolean isServerMessage = false; JsonParser parser = new JsonParser(); - JsonObject asJsonObject = parser.parse(json).getAsJsonObject(); - if(asJsonObject.has("code")) { - JsonElement codeItem = asJsonObject.get("code"); - int code = codeItem.getAsInt(); - if(code == -1) - isServerMessage = true; + JsonElement jsonElement = parser.parse(json); + if(jsonElement.isJsonObject()) { + JsonObject asJsonObject = jsonElement.getAsJsonObject(); + if(asJsonObject.has("code")) { + JsonElement codeItem = asJsonObject.get("code"); + int code = codeItem.getAsInt(); + if(code == -1) + isServerMessage = true; + } } return isServerMessage; } diff --git a/media/css/reader.css b/media/css/reader.css index 45d9d2450..fde302028 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -999,11 +999,11 @@ body { /* ================ */ -#story_titles ::-moz-selection { +.NB-feed-story-header-info ::-moz-selection { background: transparent; } -#story_titles ::selection { +.NB-feed-story-header-info ::selection { background: transparent; } @@ -9108,4 +9108,4 @@ form.opml_import_form input { } .NB-static-feedchooser .NB-feedchooser-premium-bullets li { background-color: rgba(255, 255, 255, .4); -} \ No newline at end of file +} diff --git a/templates/mail/email_base.xhtml b/templates/mail/email_base.xhtml index 5b0563f78..7de23f217 100644 --- a/templates/mail/email_base.xhtml +++ b/templates/mail/email_base.xhtml @@ -14,7 +14,9 @@ diff --git a/templates/mail/email_story_html.xhtml b/templates/mail/email_story_html.xhtml index a099875ad..01fb6369d 100644 --- a/templates/mail/email_story_html.xhtml +++ b/templates/mail/email_story_html.xhtml @@ -1,10 +1,12 @@ {% extends "mail/email_base.xhtml" %} {% load social_tags %} +{% block header %}{% endblock %} + {% block body %} -

- {{ from_name }} is sharing a story with you entitled "{{ story.story_title }}"{% if feed %} from {{ feed.feed_title }}{% endif %}: +

+ {{ from_name }} is sharing a story with you entitled "{{ story.story_title }}"{% if feed %} from {{ feed.feed_title }}{% endif %}

{% if comments %} @@ -12,7 +14,7 @@ {{ comments }}

{% endif %} -
+

{{ story.story_title }}

+ {% block header %} NewsBlur + {% endblock header %}