diff --git a/apps/reader/models.py b/apps/reader/models.py index 1ff0bf637..3e255e563 100644 --- a/apps/reader/models.py +++ b/apps/reader/models.py @@ -652,6 +652,37 @@ class UserSubscription(models.Model): class RUserStory: + @classmethod + def mark_story_hashes_read(cls, user_id, story_hashes, r=None, r2=None): + if not r: + r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL) + if not r2: + r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2) + + p = r.pipeline() + p2 = r2.pipeline() + feed_ids = set() + friend_ids = set() + + if not isinstance(story_hashes, list): + story_hashes = [story_hashes] + + for story_hash in story_hashes: + feed_id, _ = MStory.split_story_hash(story_hash) + feed_ids.add(feed_id) + + # Find other social feeds with this story to update their counts + friend_key = "F:%s:F" % (user_id) + share_key = "S:%s" % (story_hash) + friends_with_shares = [int(f) for f in r.sinter(share_key, friend_key)] + friend_ids.update(friends_with_shares) + cls.mark_read(user_id, feed_id, story_hash, social_user_ids=friends_with_shares, r=p, r2=p2) + + p.execute() + p2.execute() + + return feed_ids, friend_ids + @classmethod def mark_read(cls, user_id, story_feed_id, story_hash, social_user_ids=None, r=None, r2=None): if not r: diff --git a/apps/reader/urls.py b/apps/reader/urls.py index eb0f192bf..c5abcf024 100644 --- a/apps/reader/urls.py +++ b/apps/reader/urls.py @@ -21,6 +21,7 @@ urlpatterns = patterns('', url(r'^unread_story_hashes', views.unread_story_hashes, name='unread-story-hashes'), url(r'^mark_all_as_read', views.mark_all_as_read, name='mark-all-as-read'), url(r'^mark_story_as_read', views.mark_story_as_read, name='mark-story-as-read'), + url(r'^mark_story_hashes_as_read', views.mark_story_hashes_as_read, name='mark-story-hashes-as-read'), url(r'^mark_feed_stories_as_read', views.mark_feed_stories_as_read, name='mark-feed-stories-as-read'), url(r'^mark_social_stories_as_read', views.mark_social_stories_as_read, name='mark-social-stories-as-read'), url(r'^mark_story_as_unread', views.mark_story_as_unread), diff --git a/apps/reader/views.py b/apps/reader/views.py index 8dd85f835..961a0c20d 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -1016,7 +1016,38 @@ def mark_story_as_read(request): r.publish(request.user.username, 'feed:%s' % feed_id) return data + +@ajax_login_required +@json.json_view +def mark_story_hashes_as_read(request): + r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) + story_hashes = request.REQUEST.getlist('story_hash') + feed_ids, friend_ids = RUserStory.mark_story_hashes_read(request.user.pk, story_hashes) + + if friend_ids: + socialsubs = MSocialSubscription.objects.filter( + user_id=request.user.pk, + subscription_user_id__in=friend_ids) + for socialsub in socialsubs: + if not socialsub.needs_unread_recalc: + socialsub.needs_unread_recalc = True + socialsub.save() + r.publish(request.user.username, 'social:%s' % socialsub.subscription_user_id) + + + # Also count on original subscription + for feed_id in feed_ids: + usersubs = UserSubscription.objects.filter(user=request.user.pk, feed=feed_id) + if usersubs: + usersub = usersubs[0] + if not usersub.needs_unread_recalc: + usersub.needs_unread_recalc = True + usersub.save() + r.publish(request.user.username, 'feed:%s' % feed_id) + + return dict(code=1, story_hashes=story_hashes, feed_ids=feed_ids, friend_user_ids=friend_ids) + @ajax_login_required @json.json_view def mark_feed_stories_as_read(request): diff --git a/clients/android/NewsBlur/src/com/newsblur/fragment/FeedItemListFragment.java b/clients/android/NewsBlur/src/com/newsblur/fragment/FeedItemListFragment.java index 2e9210366..7a299b96c 100644 --- a/clients/android/NewsBlur/src/com/newsblur/fragment/FeedItemListFragment.java +++ b/clients/android/NewsBlur/src/com/newsblur/fragment/FeedItemListFragment.java @@ -87,10 +87,11 @@ public class FeedItemListFragment extends StoryItemListFragment implements Loade String[] groupFrom = new String[] { DatabaseConstants.STORY_TITLE, DatabaseConstants.STORY_AUTHORS, DatabaseConstants.STORY_READ, DatabaseConstants.STORY_SHORTDATE, DatabaseConstants.STORY_INTELLIGENCE_AUTHORS }; int[] groupTo = new int[] { R.id.row_item_title, R.id.row_item_author, R.id.row_item_title, R.id.row_item_date, R.id.row_item_sidebar }; - getLoaderManager().initLoader(ITEMLIST_LOADER , null, this); - + // create the adapter before starting the loader, since the callback updates the adapter adapter = new FeedItemsAdapter(getActivity(), feed, R.layout.row_item, storiesCursor, groupFrom, groupTo, CursorAdapter.FLAG_REGISTER_CONTENT_OBSERVER); + getLoaderManager().initLoader(ITEMLIST_LOADER , null, this); + itemList.setOnScrollListener(this); adapter.setViewBinder(new FeedItemViewBinder(getActivity())); diff --git a/clients/android/NewsBlur/src/com/newsblur/util/FeedUtils.java b/clients/android/NewsBlur/src/com/newsblur/util/FeedUtils.java index 0b26de835..297082de3 100644 --- a/clients/android/NewsBlur/src/com/newsblur/util/FeedUtils.java +++ b/clients/android/NewsBlur/src/com/newsblur/util/FeedUtils.java @@ -90,15 +90,17 @@ public class FeedUtils { ArrayList updateOps = new ArrayList(); for (Story story : stories) { + // ops for the local DB appendStoryReadOperations(story, updateOps); + // API call to ensure the story is marked read in the context of a feed + storiesJson.put(story.feedId, story.storyHash); + // API call to ensure the story is marked read in the context of a social share if (story.socialUserId != null) { - // TODO: some stories returned by /social/river_stories seem to have neither a - // socialUserId nor a sourceUserId, so they accidentally get submitted non- - // socially. If the API fixes this before we ditch social-specific logic, - // we can fix that bug right here. - putMapHeirarchy(socialStories, story.socialUserId, story.feedId, story.id); - } else { - storiesJson.put(story.feedId, story.id); + putMapHeirarchy(socialStories, story.socialUserId, story.feedId, story.storyHash); + } else if ((story.friendUserIds != null) && (story.friendUserIds.length > 0) && (story.friendUserIds[0] != null)) { + putMapHeirarchy(socialStories, story.friendUserIds[0], story.feedId, story.storyHash); + } else if ((story.sharedUserIds != null) && (story.sharedUserIds.length > 0) && (story.sharedUserIds[0] != null)) { + putMapHeirarchy(socialStories, story.sharedUserIds[0], story.feedId, story.storyHash); } } diff --git a/fabfile.py b/fabfile.py index 5ff2ffde1..479dd6965 100644 --- a/fabfile.py +++ b/fabfile.py @@ -82,8 +82,8 @@ def list_do(): def host(*names): env.hosts = [] hostnames = do(split=True) - for role in hostnames.keys(): - for host in hostnames[role]: + for role, hosts in hostnames.items(): + for host in hosts: if isinstance(host, dict) and host['name'] in names: env.hosts.append(host['address']) print " ---> Using %s as hosts" % env.hosts @@ -94,7 +94,6 @@ def host(*names): def server(): env.NEWSBLUR_PATH = "/srv/newsblur" - env.SECRETS_PATH = "/srv/secrets-newsblur" env.VENDOR_PATH = "/srv/code" def do(split=False): @@ -262,29 +261,58 @@ def setup_task_image(): # = Setup - Common = # ================== +def done(): + print "\n\n\n\n-----------------------------------------------------" + print "\n\n %s IS SUCCESSFULLY BOOTSTRAPPED" % env.host_string + print "\n\n-----------------------------------------------------\n\n\n\n" + def setup_installs(): + packages = [ + 'build-essential', + 'gcc', + 'scons', + 'libreadline-dev', + 'sysstat', + 'iotop', + 'git', + 'python-dev', + 'locate', + 'python-software-properties', + 'software-properties-common', + 'libpcre3-dev', + 'libncurses5-dev', + 'libdbd-pg-perl', + 'libssl-dev', + 'make', + 'pgbouncer', + 'python-setuptools', + 'python-psycopg2', + 'libyaml-0-2', + 'python-yaml', + 'python-numpy', + 'python-scipy', + 'curl', + 'monit', + 'ufw', + 'libjpeg8', + 'libjpeg62-dev', + 'libfreetype6', + 'libfreetype6-dev', + 'python-imaging', + ] sudo('apt-get -y update') sudo('DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes upgrade') - sudo('DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install build-essential gcc scons libreadline-dev sysstat iotop git python-dev locate python-software-properties software-properties-common libpcre3-dev libncurses5-dev libdbd-pg-perl libssl-dev make pgbouncer python-setuptools python-psycopg2 libyaml-0-2 python-yaml python-numpy python-scipy curl monit ufw libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev python-imaging') + sudo('DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install %s' % ' '.join(packages)) with settings(warn_only=True): sudo("ln -s /usr/lib/x86_64-linux-gnu/libjpeg.so /usr/lib") sudo("ln -s /usr/lib/x86_64-linux-gnu/libfreetype.so /usr/lib") sudo("ln -s /usr/lib/x86_64-linux-gnu/libz.so /usr/lib") - # sudo('add-apt-repository ppa:pitti/postgresql') - # sudo('apt-get -y update') - # run('curl -O http://peak.telecommunity.com/dist/ez_setup.py') - # sudo('python ez_setup.py -U setuptools && rm ez_setup.py') with settings(warn_only=True): sudo('mkdir -p %s' % env.VENDOR_PATH) sudo('chown %s.%s %s' % (env.user, env.user, env.VENDOR_PATH)) -def done(): - print "\n\n-----------------------------------------------------" - print "\n\n %s IS SUCCESSFULLY BOOTSTRAPPED" % env.host_string - print "\n\n-----------------------------------------------------\n\n" - def change_shell(): sudo('apt-get -y install zsh') with settings(warn_only=True): @@ -302,8 +330,8 @@ def setup_user(): run('echo `cat authorized_keys` >> ~/.ssh/authorized_keys') run('rm authorized_keys') -def add_machine_to_ssh(): - put("~/.ssh/id_dsa.pub", "local_keys") +def copy_ssh_keys(): + put(os.path.join(env.SECRETS_PATH, 'keys/newsblur.key.pub'), "local_keys") run("echo `cat local_keys` >> .ssh/authorized_keys") run("rm local_keys") @@ -386,11 +414,12 @@ def setup_supervisor(): @parallel def setup_hosts(): - put('../secrets-newsblur/configs/hosts', '/etc/hosts', use_sudo=True) + put(os.path.join(env.SECRETS_PATH, 'configs/hosts'), '/etc/hosts', use_sudo=True) def config_pgbouncer(): put('config/pgbouncer.conf', '/etc/pgbouncer/pgbouncer.ini', use_sudo=True) - put('../secrets-newsblur/configs/pgbouncer_auth.conf', '/etc/pgbouncer/userlist.txt', use_sudo=True) + put(os.path.join(env.SECRETS_PATH, 'configs/pgbouncer_auth.conf'), + '/etc/pgbouncer/userlist.txt', use_sudo=True) sudo('echo "START=1" > /etc/default/pgbouncer') sudo('su postgres -c "/etc/init.d/pgbouncer stop"', pty=False) with settings(warn_only=True): @@ -580,17 +609,17 @@ def config_node(): @parallel def copy_app_settings(): - put('../secrets-newsblur/settings/app_settings.py', '%s/local_settings.py' % env.NEWSBLUR_PATH) + put(os.path.join(env.SECRETS_PATH, 'settings/app_settings.py'), + '%s/local_settings.py' % env.NEWSBLUR_PATH) run('echo "\nSERVER_NAME = \\\\"`hostname`\\\\"" >> %s/local_settings.py' % env.NEWSBLUR_PATH) def copy_certificates(): cert_path = '%s/config/certificates/' % env.NEWSBLUR_PATH run('mkdir -p %s' % cert_path) - put('../secrets-newsblur/certificates/newsblur.com.crt', cert_path) - put('../secrets-newsblur/certificates/newsblur.com.key', cert_path) + put(os.path.join(env.SECRETS_PATH, 'certificates/newsblur.com.crt'), cert_path) + put(os.path.join(env.SECRETS_PATH, 'certificates/newsblur.com.key'), cert_path) run('cat %s/newsblur.com.crt > %s/newsblur.pem' % (cert_path, cert_path)) run('cat %s/newsblur.com.key >> %s/newsblur.pem' % (cert_path, cert_path)) - # put('../secrets-newsblur/certificates/comodo/EssentialSSLCA_2.crt', '%s/config/certificates/intermediate.crt' % env.NEWSBLUR_PATH) @parallel def maintenance_on(): @@ -621,7 +650,8 @@ def setup_haproxy(debug=False): if debug: put('config/debug_haproxy.conf', '/etc/haproxy/haproxy.cfg', use_sudo=True) else: - put('../secrets-newsblur/configs/haproxy.conf', '/etc/haproxy/haproxy.cfg', use_sudo=True) + put(os.path.join(env.SECRETS_PATH, 'configs/haproxy.conf'), + '/etc/haproxy/haproxy.cfg', use_sudo=True) sudo('echo "ENABLED=1" > /etc/default/haproxy') cert_path = "%s/config/certificates" % env.NEWSBLUR_PATH run('cat %s/newsblur.com.crt > %s/newsblur.pem' % (cert_path, cert_path)) @@ -636,7 +666,8 @@ def config_haproxy(debug=False): if debug: put('config/debug_haproxy.conf', '/etc/haproxy/haproxy.cfg', use_sudo=True) else: - put('../secrets-newsblur/configs/haproxy.conf', '/etc/haproxy/haproxy.cfg', use_sudo=True) + put(os.path.join(env.SECRETS_PATH, 'configs/haproxy.conf'), + '/etc/haproxy/haproxy.cfg', use_sudo=True) sudo('/etc/init.d/haproxy reload') def upgrade_django(): @@ -787,7 +818,8 @@ def setup_mongo_mongos(): def setup_mongo_mms(): pull() - put('../secrets-newsblur/settings/mongo_mms_settings.py', '%s/vendor/mms-agent/settings.py' % env.NEWSBLUR_PATH) + put(os.path.join(env.SECRETS_PATH, 'settings/mongo_mms_settings.py'), + '%s/vendor/mms-agent/settings.py' % env.NEWSBLUR_PATH) with cd(env.NEWSBLUR_PATH): put('config/supervisor_mongomms.conf', '/etc/supervisor/conf.d/mongomms.conf', use_sudo=True) sudo('supervisorctl reread') @@ -935,7 +967,8 @@ def copy_task_settings(): host = env.host_string.split('.', 2)[0] with settings(warn_only=True): - put('../secrets-newsblur/settings/task_settings.py', '%s/local_settings.py' % env.NEWSBLUR_PATH) + put(os.path.join(env.SECRETS_PATH, 'settings/task_settings.py'), + '%s/local_settings.py' % env.NEWSBLUR_PATH) run('echo "\nSERVER_NAME = \\\\"%s\\\\"" >> %s/local_settings.py' % (host, env.NEWSBLUR_PATH)) # ========================= @@ -1015,9 +1048,7 @@ def add_user_to_do(): run('rm -fr ~%s/.ssh/id_dsa*' % (repo_user)) run('ssh-keygen -t dsa -f ~%s/.ssh/id_dsa -N ""' % (repo_user)) run('touch ~%s/.ssh/authorized_keys' % (repo_user)) - put("~/.ssh/id_dsa.pub", "authorized_keys") - run('echo `cat authorized_keys` >> ~%s/.ssh/authorized_keys' % (repo_user)) - run('rm authorized_keys') + copy_ssh_keys() run('chown %s.%s -R ~%s/.ssh' % (repo_user, repo_user, repo_user)) env.user = repo_user diff --git a/media/js/newsblur/common/assetmodel.js b/media/js/newsblur/common/assetmodel.js index f6be836b6..b64a45052 100644 --- a/media/js/newsblur/common/assetmodel.js +++ b/media/js/newsblur/common/assetmodel.js @@ -143,6 +143,32 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({ $.isFunction(callback) && callback(read); }, + mark_story_hash_as_read: function(story, callback) { + var self = this; + var read = story.get('read_status'); + + if (!story.get('read_status')) { + story.set('read_status', 1); + + if (NEWSBLUR.Globals.is_authenticated) { + if (!('hashes' in this.queued_read_stories)) { this.queued_read_stories['hashes'] = []; } + this.queued_read_stories['hashes'].push(story.get('story_hash')); + // NEWSBLUR.log(['Marking Read', this.queued_read_stories, story.id]); + + this.make_request('/reader/mark_story_hashes_as_read', { + story_hash: this.queued_read_stories['hashes'] + }, null, null, { + 'ajax_group': 'queue_clear', + 'beforeSend': function() { + self.queued_read_stories = {}; + } + }); + } + } + + $.isFunction(callback) && callback(read); + }, + mark_social_story_as_read: function(story, social_feed, callback) { var self = this; var feed_id = story.get('story_feed_id'); diff --git a/settings.py b/settings.py index f51936935..5857de35e 100644 --- a/settings.py +++ b/settings.py @@ -379,7 +379,7 @@ CELERYBEAT_SCHEDULE = { }, 'activate-next-new-user': { 'task': 'activate-next-new-user', - 'schedule': datetime.timedelta(minutes=2), + 'schedule': datetime.timedelta(minutes=10), 'options': {'queue': 'beat_tasks'}, }, } diff --git a/utils/ssh.sh b/utils/ssh.sh index adb802627..8798b7901 100755 --- a/utils/ssh.sh +++ b/utils/ssh.sh @@ -20,4 +20,4 @@ WHITE='\033[01;37m' ipaddr=`python /srv/newsblur/utils/hostname_ssh.py $1` printf "\n ${BLUE}---> ${LBLUE}Connecting to ${LGREEN}$1${BLUE} / ${LRED}$ipaddr${BLUE} <--- ${RESTORE}\n\n" -ssh $ipaddr \ No newline at end of file +ssh -i ~/projects/secrets-newsblur/keys/newsblur.key $ipaddr \ No newline at end of file