diff --git a/apps/profile/models.py b/apps/profile/models.py index 634e20712..4b6a4b897 100644 --- a/apps/profile/models.py +++ b/apps/profile/models.py @@ -211,14 +211,7 @@ class Profile(models.Model): print " ---> %s payments" % len(payment_history) if most_recent_payment_date: - payment_gap = 0 - # If user lapsed and has no gap b/w last payment and expiration, - # they only get a full year. Otherwise, give them the gap. - if (self.premium_expire and - self.premium_expire > datetime.datetime.now() and - self.premium_expire > most_recent_payment_date): - payment_gap = (self.premium_expire - most_recent_payment_date).days - self.premium_expire = most_recent_payment_date + datetime.timedelta(days=365+payment_gap) + self.premium_expire = most_recent_payment_date + datetime.timedelta(days=365) self.save() def cancel_premium(self): diff --git a/apps/rss_feeds/migrations/0063_story_hash.py b/apps/rss_feeds/migrations/0063_story_hash.py deleted file mode 100644 index a1612e0a5..000000000 --- a/apps/rss_feeds/migrations/0063_story_hash.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- coding: utf-8 -*- -from south.v2 import DataMigration - -class Migration(DataMigration): - - def forwards(self, orm): - from apps.rss_feeds.models import MStory, Feed - import time - - batch = 0 - for f in xrange(Feed.objects.latest('pk').pk): - if f < batch*100000: continue - start = time.time() - try: - try: - feed = Feed.get_by_id(f) - except Feed.DoesNotExist: - continue - if not feed: continue - cp1 = time.time() - start - if feed.active_premium_subscribers < 1: continue - stories = MStory.objects.filter(story_feed_id=feed.pk, story_hash__exists=False) - cp2 = time.time() - start - for story in stories: story.save() - cp3 = time.time() - start - print "%3s stories: %s (%s/%s/%s)" % (stories.count(), feed, round(cp1, 2), round(cp2, 2), round(cp3, 2)) - except Exception, e: - print " ***> (%s) %s" % (f, e) - - - - def backwards(self, orm): - "Write your backwards methods here." - - models = { - 'rss_feeds.duplicatefeed': { - 'Meta': {'object_name': 'DuplicateFeed'}, - 'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '764', 'db_index': 'True'}), - 'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}), - 'duplicate_link': ('django.db.models.fields.CharField', [], {'max_length': '764', 'null': 'True', 'db_index': 'True'}), - 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) - }, - 'rss_feeds.feed': { - 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"}, - 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), - 'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), - 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), - 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), - 'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}), - 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}), - 'errors_since_good': ('django.db.models.fields.IntegerField', [], {'default': '0'}), - 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), - 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}), - 'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}), - 'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '764', 'db_index': 'True'}), - 'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), - 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}), - 'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}), - 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), - 'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), - 'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), - 'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), - 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), - 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), - 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), - 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), - 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), - 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), - 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), - 's3_icon': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), - 's3_page': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), - 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}) - }, - 'rss_feeds.feeddata': { - 'Meta': {'object_name': 'FeedData'}, - 'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), - 'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), - 'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), - 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), - 'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) - } - } - - complete_apps = ['rss_feeds'] - symmetrical = True diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index b7e5df17f..126f05c50 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1222,9 +1222,11 @@ class Feed(models.Model): # 1 update per day = 1 hours # 10 updates = 20 minutes updates_per_day_delay = 3 * 60 / max(.25, ((max(0, self.active_subscribers)**.2) - * (updates_per_month**0.35))) - if self.premium_subscribers > 0: - updates_per_day_delay /= min(self.active_subscribers+self.premium_subscribers, 5) + * (updates_per_month**0.25))) + if self.active_premium_subscribers > 0: + updates_per_day_delay /= min(self.active_subscribers+self.active_premium_subscribers, 4) + updates_per_day_delay = int(updates_per_day_delay) + # Lots of subscribers = lots of updates # 24 hours for 0 subscribers. # 4 hours for 1 subscriber. @@ -1234,6 +1236,7 @@ class Feed(models.Model): subscriber_bonus = 12 * 60 / max(.167, max(0, self.active_subscribers)**3) if self.premium_subscribers > 0: subscriber_bonus /= min(self.active_subscribers+self.premium_subscribers, 5) + subscriber_bonus = int(subscriber_bonus) slow_punishment = 0 if self.num_subscribers <= 1: @@ -1259,16 +1262,21 @@ class Feed(models.Model): total = 60*24*30 if verbose: - print "[%s] %s (%s/%s/%s/%s), %s, %s: %s" % (self, updates_per_day_delay, - self.num_subscribers, self.active_subscribers, - self.premium_subscribers, self.active_premium_subscribers, - subscriber_bonus, slow_punishment, total) + logging.debug(" ---> [%-30s] Fetched every %sm (%s+%s+%s) Subs: %s/%s Stories: %s/%s" % ( + unicode(self)[:30], total, + updates_per_day_delay, + subscriber_bonus, + slow_punishment, + self.num_subscribers, + self.active_premium_subscribers, + self.average_stories_per_month, + self.stories_last_month)) random_factor = random.randint(0, total) / 4 return total, random_factor*8 - def set_next_scheduled_update(self, verbose=True): - total, random_factor = self.get_next_scheduled_update(force=True, verbose=False) + def set_next_scheduled_update(self, verbose=False, skip_scheduling=False): + total, random_factor = self.get_next_scheduled_update(force=True, verbose=verbose) if self.errors_since_good: total = total * self.errors_since_good @@ -1281,7 +1289,8 @@ class Feed(models.Model): minutes = total + random_factor) self.min_to_decay = total - self.next_scheduled_update = next_scheduled_update + if not skip_scheduling: + self.next_scheduled_update = next_scheduled_update self.save() diff --git a/apps/rss_feeds/tasks.py b/apps/rss_feeds/tasks.py index 4911f894e..8340d1932 100644 --- a/apps/rss_feeds/tasks.py +++ b/apps/rss_feeds/tasks.py @@ -14,6 +14,7 @@ class TaskFeeds(Task): from apps.rss_feeds.models import Feed settings.LOG_TO_STREAM = True now = datetime.datetime.utcnow() + start = time.time() # Active feeds popular_feeds = Feed.objects.filter( @@ -28,7 +29,7 @@ class TaskFeeds(Task): next_scheduled_update__lte=now, active=True, active_subscribers__gte=1 - ).order_by('?')[:200] + ).order_by('?')[:600] active_count = feeds.count() # Force refresh feeds @@ -71,7 +72,9 @@ class TaskFeeds(Task): Feed.task_feeds(refresh_feeds, verbose=False) if inactive_feeds: Feed.task_feeds(inactive_feeds, verbose=False) if old_feeds: Feed.task_feeds(old_feeds, verbose=False) - + + logging.debug(" ---> ~FBTasking took %.2s seconds" % (time.time() - start)) + class UpdateFeeds(Task): name = 'update-feeds' diff --git a/apps/rss_feeds/views.py b/apps/rss_feeds/views.py index 5a40c31a4..dcec03377 100644 --- a/apps/rss_feeds/views.py +++ b/apps/rss_feeds/views.py @@ -134,6 +134,8 @@ def load_feed_statistics(request, feed_id): user = get_user(request) stats = dict() feed = get_object_or_404(Feed, pk=feed_id) + feed.count_subscribers() + feed.set_next_scheduled_update(verbose=True, skip_scheduling=True) feed.save_feed_story_history_statistics() feed.save_classifier_counts() diff --git a/apps/social/migrations/0007_prune_redis_hash.py b/apps/social/migrations/0007_prune_redis_hash.py new file mode 100644 index 000000000..d17e7070e --- /dev/null +++ b/apps/social/migrations/0007_prune_redis_hash.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models +import redis +from django.conf import settings + +class Migration(DataMigration): + + def forwards(self, orm): + r = redis.Redis(connection_pool=settings.REDIS_POOL) + keys = r.keys("*:*:????????????????????????????????????????") + print " ---> %s keys" % len(keys) + for key in keys: + print "Deleting %s" % key + r.delete(key) + + def backwards(self, orm): + "Write your backwards methods here." + + models = { + + } + + complete_apps = ['social'] + symmetrical = True diff --git a/config/mongodb.prod.conf b/config/mongodb.prod.conf index dc0520d41..7556ad773 100644 --- a/config/mongodb.prod.conf +++ b/config/mongodb.prod.conf @@ -16,6 +16,8 @@ logappend=true slowms=100 +syncdelay=5 + rest = true #profile = 2 # Enables periodic logging of CPU utilization and I/O wait diff --git a/config/munin/mongo_indexsize b/config/munin/mongo_indexsize new file mode 100644 index 000000000..15898bb14 --- /dev/null +++ b/config/munin/mongo_indexsize @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# vim: set sts=4 sw=4 encoding=utf-8 + +# Copyright (c) 2010, Rene Jochum +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the Rene Jochum nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from pymongo import Connection +import os + +settings_host = os.environ.get("host", "127.0.0.1") +settings_port = 27017 +settings_db = 'newsblur' + +def getCollstats(): + global settings_host, settings_port, settings_db, settings_user, settings_password + print locals(), settings_host, settings_port + if settings_user and settings_password: + settings_host = "%s:%s@%s" % (settings_user, settings_password, settings_host) + con = Connection(settings_host, int(settings_port), slave_okay=True) + db = con[settings_db] + + for coll in db.collection_names(): + if coll.startswith('system.'): + continue + stats = db.command("collstats", coll) + yield ("%s_size" % coll.replace('.', '_'), long(stats['totalIndexSize']),) + + con.disconnect() + + +def doData(): + for coll, stats in getCollstats(): + print "%s.value %s" % (coll, stats) + + +def doConfig(): + + print "graph_title MongoDB collection index sizes" + print "graph_args --base 1024 -l 0" + print "graph_vlabel Kb" + print "graph_category MongoDB" + print "graph_total total" + + for k,v in getCollstats(): + print "%s.label %s" % (k, k) + print "%s.min 0" % k + print "%s.draw LINE1" % k + + +if __name__ == "__main__": + from sys import argv + from os import environ + + # Could be done by a for loop + # but i think if's are faster + if 'HOST' in environ: + settings_host = environ['HOST'] + if 'PORT' in environ: + settings_port = environ['PORT'] + if 'DB' in environ: + settings_db = environ['DB'] + if 'user' in environ: + settings_user = environ['user'] + if 'password' in environ: + settings_password = environ['password'] + print locals() + if len(argv) > 1 and argv[1] == "config": + doConfig() + else: + doData() diff --git a/config/munin/redis_size b/config/munin/redis_size index 592d4a6ac..f1f772628 100644 --- a/config/munin/redis_size +++ b/config/munin/redis_size @@ -18,8 +18,8 @@ class NBMuninGraph(MuninGraph): } stats = self.stats - graph.update(dict((("%s.label" % s['_id'], s['_id']) for s in stats))) - graph.update(dict((("%s.draw" % s['_id'], 'LINE1') for s in stats))) + graph.update(dict((("%s.label" % s, s) for s in stats.keys()))) + graph.update(dict((("%s.draw" % s, 'LINE1') for s in stats.keys()))) return graph diff --git a/config/nginx.newsblur.conf b/config/nginx.newsblur.conf index 601eaf3cb..f17feaff5 100644 --- a/config/nginx.newsblur.conf +++ b/config/nginx.newsblur.conf @@ -2,9 +2,15 @@ upstream app_server { server 127.0.0.1:8000 fail_timeout=10 max_fails=3 ; } +upstream icon_server { + server 127.0.0.1:3030 fail_timeout=2 max_fails=3; + server 127.0.0.1:8000 backup; +} + + server { listen 80; - # listen 443 default_server ssl; + listen 443 default_server ssl; # ssl on; ssl_certificate /srv/newsblur/config/certificates/newsblur.com.crt; @@ -80,8 +86,15 @@ server { alias /srv/newsblur/media/robots.txt; } + location /munin/static/ { + alias /etc/munin/static/; + } + location /munin/ { - alias /var/cache/munin/www/; + fastcgi_split_path_info ^(/munin)(.*); + fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_pass unix:/var/run/munin/fcgi-html.sock; + include fastcgi_params; } location ^~ /cgi-bin/munin-cgi-graph/ { @@ -92,7 +105,12 @@ server { } location ^~ /rss_feeds/icon/ { - rewrite ^(.*)$ /media/img/icons/silk/world.png; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + + proxy_pass http://icon_server; } location / { diff --git a/config/requirements.txt b/config/requirements.txt index b17408e7f..3f6b23be9 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -9,10 +9,10 @@ django-compress==1.0.1 django-extensions==1.1.1 django-mailgun==0.2.1 django-redis-sessions==0.3.1 +django-redis-cache==0.9.7 django-ses==0.4.1 django-subdomains==2.0.3 Django>=1.5,<1.6 -django_ses Fabric==1.6.0 gunicorn==0.17.2 httplib2==0.8 @@ -34,6 +34,7 @@ PyYAML==3.10 raven==3.1.17 readline==6.2.4.1 redis==2.7.2 +hiredis==0.1.1 requests==1.1.0 seacucumber==1.5 South==0.7.6 diff --git a/config/spawn_fcgi_munin_graph.conf b/config/spawn_fcgi_munin_graph.conf index e2e97aa0d..e851c4667 100644 --- a/config/spawn_fcgi_munin_graph.conf +++ b/config/spawn_fcgi_munin_graph.conf @@ -27,7 +27,7 @@ NAME=spawn-fcgi-munin-graph PID_FILE=/var/run/munin/fcgi-graph.pid SOCK_FILE=/var/run/munin/fcgi-graph.sock DAEMON=/usr/bin/spawn-fcgi -DAEMON_OPTS="-s $SOCK_FILE -U www-data -u www-data -g www-data /usr/lib/cgi-bin/munin-cgi-graph -P $PID_FILE" +DAEMON_OPTS="-s $SOCK_FILE -U nginx -u nginx -g www-data /usr/lib/cgi-bin/munin-cgi-graph -P $PID_FILE" # -------------------------------------------------------------- # No edits necessary beyond this line diff --git a/config/spawn_fcgi_munin_html.conf b/config/spawn_fcgi_munin_html.conf new file mode 100644 index 000000000..4e26fd66b --- /dev/null +++ b/config/spawn_fcgi_munin_html.conf @@ -0,0 +1,119 @@ +#! /bin/sh + +### BEGIN INIT INFO +# Provides: spawn-fcgi-munin-html +# Required-Start: $all +# Required-Stop: $all +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Description: starts FastCGI for Munin-Html +### END INIT INFO +# -------------------------------------------------------------- +# Munin-CGI-HTML Spawn-FCGI Startscript by Julien Schmidt +# eMail: munin-trac at julienschmidt.com +# www: http://www.julienschmidt.com +# -------------------------------------------------------------- +# Install: +# 1. Copy this file to /etc/init.d +# 2. Edit the variables below +# 3. run "update-rc.d spawn-fcgi-munin-html defaults" +# -------------------------------------------------------------- +# Last Update: 13. November 2012 +# +# Please change the following variables: + +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +NAME=spawn-fcgi-munin-html +PID_FILE=/var/run/munin/fcgi-html.pid +SOCK_FILE=/var/run/munin/fcgi-html.sock +DAEMON=/usr/bin/spawn-fcgi +DAEMON_OPTS="-s $SOCK_FILE -U nginx -u nginx -g www-data /usr/lib/cgi-bin/munin-cgi-html -P $PID_FILE" + +# -------------------------------------------------------------- +# No edits necessary beyond this line +# -------------------------------------------------------------- + +if [ ! -x $DAEMON ]; then + echo "File not found or is not executable: $DAEMON!" + exit 0 +fi + +status() { + if [ ! -r $PID_FILE ]; then + return 1 + fi + + FCGI_PID=`cat $PID_FILE` + if [ -z "${FCGI_PID}" ]; then + return 1 + fi + + FCGI_RUNNING=`ps -p ${FCGI_PID} | grep ${FCGI_PID}` + if [ -z "${FCGI_RUNNING}" ]; then + return 1 + fi + + return 0 +} + +start() { + if status; then + echo "FCGI is already running!" + exit 1 + else + $DAEMON $DAEMON_OPTS + fi +} + +stop () { + if ! status; then + echo "No PID-file at $PID_FILE found or PID not valid. Maybe not running" + exit 1 + fi + + # Kill process + kill -9 `cat $PID_FILE` + + # Remove PID-file + rm -f $PID_FILE + + # Remove Sock-File + rm -f $SOCK_FILE +} + +case "$1" in + start) + echo "Starting $NAME: " + start + echo "... DONE" + ;; + + stop) + echo "Stopping $NAME: " + stop + echo "... DONE" + ;; + + force-reload|restart) + echo "Stopping $NAME: " + stop + echo "Starting $NAME: " + start + echo "... DONE" + ;; + + status) + if status; then + echo "FCGI is RUNNING" + else + echo "FCGI is NOT RUNNING" + fi + ;; + + *) + echo "Usage: $0 {start|stop|restart|status}" + exit 1 + ;; +esac + +exit 0 \ No newline at end of file diff --git a/fabfile.py b/fabfile.py index 308ddbef0..b9b424427 100644 --- a/fabfile.py +++ b/fabfile.py @@ -154,7 +154,7 @@ def deploy_node(): with cd(env.NEWSBLUR_PATH): run('sudo supervisorctl restart node_unread') run('sudo supervisorctl restart node_unread_ssl') - # run('sudo supervisorctl restart node_favicons') + run('sudo supervisorctl restart node_favicons') def gunicorn_restart(): restart_gunicorn() @@ -637,7 +637,7 @@ def configure_node(): sudo('rm -fr /etc/supervisor/conf.d/node.conf') put('config/supervisor_node_unread.conf', '/etc/supervisor/conf.d/node_unread.conf', use_sudo=True) put('config/supervisor_node_unread_ssl.conf', '/etc/supervisor/conf.d/node_unread_ssl.conf', use_sudo=True) - # put('config/supervisor_node_favicons.conf', '/etc/supervisor/conf.d/node_favicons.conf', use_sudo=True) + put('config/supervisor_node_favicons.conf', '/etc/supervisor/conf.d/node_favicons.conf', use_sudo=True) sudo('supervisorctl reload') @parallel @@ -737,7 +737,8 @@ def setup_db_firewall(): for ip in set(env.roledefs['app'] + env.roledefs['dbdo'] + env.roledefs['dev'] + - env.roledefs['debug']): + env.roledefs['debug'] + + env.roledefs['task']): sudo('ufw allow proto tcp from %s to any port %s' % ( ip, ','.join(map(str, ports)) @@ -824,16 +825,29 @@ def setup_redis(): sudo('/etc/init.d/redis start') def setup_munin(): - sudo('apt-get update') + # sudo('apt-get update') sudo('apt-get install -y munin munin-node munin-plugins-extra spawn-fcgi') put('config/munin.conf', '/etc/munin/munin.conf', use_sudo=True) put('config/spawn_fcgi_munin_graph.conf', '/etc/init.d/spawn_fcgi_munin_graph', use_sudo=True) + put('config/spawn_fcgi_munin_html.conf', '/etc/init.d/spawn_fcgi_munin_html', use_sudo=True) sudo('chmod u+x /etc/init.d/spawn_fcgi_munin_graph') + sudo('chmod u+x /etc/init.d/spawn_fcgi_munin_html') with settings(warn_only=True): + sudo('chown nginx.www-data munin-cgi*') + with settings(warn_only=True): + sudo('/etc/init.d/spawn_fcgi_munin_graph stop') sudo('/etc/init.d/spawn_fcgi_munin_graph start') sudo('update-rc.d spawn_fcgi_munin_graph defaults') + sudo('/etc/init.d/spawn_fcgi_munin_html stop') + sudo('/etc/init.d/spawn_fcgi_munin_html start') + sudo('update-rc.d spawn_fcgi_munin_html defaults') sudo('/etc/init.d/munin-node restart') - + with settings(warn_only=True): + sudo('chown nginx.www-data munin-cgi*') + with settings(warn_only=True): + sudo('/etc/init.d/spawn_fcgi_munin_graph start') + sudo('/etc/init.d/spawn_fcgi_munin_html start') + def setup_db_munin(): sudo('cp -frs %s/config/munin/mongo* /etc/munin/plugins/' % env.NEWSBLUR_PATH) diff --git a/local_settings.py.template b/local_settings.py.template index ec9c74275..87c47fd04 100644 --- a/local_settings.py.template +++ b/local_settings.py.template @@ -23,10 +23,21 @@ DEBUG_ASSETS = DEBUG MEDIA_URL = '/media/' SECRET_KEY = 'YOUR SECRET KEY' -CACHE_BACKEND = 'dummy:///' +# CACHE_BACKEND = 'dummy:///' # CACHE_BACKEND = 'locmem:///' # CACHE_BACKEND = 'memcached://127.0.0.1:11211' +CACHES = { + 'default': { + 'BACKEND': 'redis_cache.RedisCache', + 'LOCATION': '127.0.0.1:6379', + 'OPTIONS': { + 'DB': 6, + 'PARSER_CLASS': 'redis.connection.HiredisParser' + }, + }, +} + EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' # Set this to the username that is shown on the homepage to unauthenticated users. diff --git a/media/android/NewsBlur/src/com/newsblur/activity/NewsBlurApplication.java b/media/android/NewsBlur/src/com/newsblur/activity/NewsBlurApplication.java index f96681f22..ce606976e 100644 --- a/media/android/NewsBlur/src/com/newsblur/activity/NewsBlurApplication.java +++ b/media/android/NewsBlur/src/com/newsblur/activity/NewsBlurApplication.java @@ -8,11 +8,12 @@ public class NewsBlurApplication extends Application { ImageLoader imageLoader; - public NewsBlurApplication() { - super(); + @Override + public void onCreate() { + super.onCreate(); imageLoader = new ImageLoader(this); } - + public ImageLoader getImageLoader() { return imageLoader; } diff --git a/media/android/NewsBlur/src/com/newsblur/activity/Profile.java b/media/android/NewsBlur/src/com/newsblur/activity/Profile.java index 3c413ae60..548b5af53 100644 --- a/media/android/NewsBlur/src/com/newsblur/activity/Profile.java +++ b/media/android/NewsBlur/src/com/newsblur/activity/Profile.java @@ -91,9 +91,13 @@ public class Profile extends SherlockFragmentActivity { } else { apiManager.updateUserProfile(); user = PrefsUtils.getUserDetails(Profile.this); - profileResponse = apiManager.getUser(user.id); - if (profileResponse != null) { - activities = profileResponse.activities; + // check user.id has been set. If previous attempts to update the user details + // have failed then user.id == null would cause a force close + if (user.id != null) { + profileResponse = apiManager.getUser(user.id); + if (profileResponse != null) { + activities = profileResponse.activities; + } } } return null; @@ -103,8 +107,10 @@ public class Profile extends SherlockFragmentActivity { protected void onPostExecute(Void result) { if (user != null && detailsFragment != null && activitiesFragment != null) { detailsFragment.setUser(Profile.this, user, TextUtils.isEmpty(userId)); - // TODO still sometimes causes a force close - is activities null ? - activitiesFragment.setActivitiesAndUser(Profile.this, activities, user); + // activities could be null if no profile response was received + if (activities != null) { + activitiesFragment.setActivitiesAndUser(Profile.this, activities, user); + } } } } diff --git a/media/css/reader.css b/media/css/reader.css index d63809567..5a222120e 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -6142,6 +6142,15 @@ form.opml_import_form input { overflow: hidden; margin: 0 0 12px 0; } +.NB-modal-statistics .NB-statistics-update-explainer { + clear: both; + margin: 12px 24px 0; + font-size: 10px; + color: #808080; +} +.NB-modal-statistics .NB-statistics-update-explainer b { + padding-right: 8px; +} .NB-modal-statistics .NB-statistics-premium-stats { border-top: 1px solid #E0E0E0; padding: 12px 0 0; diff --git a/media/js/newsblur/reader/reader.js b/media/js/newsblur/reader/reader.js index 88fe4801b..1aa7c581f 100644 --- a/media/js/newsblur/reader/reader.js +++ b/media/js/newsblur/reader/reader.js @@ -3944,9 +3944,9 @@ } else if (force || !this.socket || !this.socket.socket.connected) { var server = window.location.protocol + '//' + window.location.hostname; var https = _.string.startsWith(window.location.protocol, 'https'); - var www = _.string.contains(window.location.href, 'www.newsblur.com'); + var dev = _.string.contains(window.location.href, 'dev.newsblur.com'); var port = https ? 443 : 80; - if (NEWSBLUR.Globals.debug || !www) { + if (NEWSBLUR.Globals.debug || dev) { port = https ? 8889 : 8888; } this.socket = this.socket || io.connect(server, { diff --git a/media/js/newsblur/reader/reader_statistics.js b/media/js/newsblur/reader/reader_statistics.js index 7273d68d4..9bd6a65bb 100644 --- a/media/js/newsblur/reader/reader_statistics.js +++ b/media/js/newsblur/reader/reader_statistics.js @@ -103,6 +103,11 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, { (data.active && $.make('div', { className: 'NB-statistics-count' }, ' ' + (data['next_update'] && ('in ' + data['next_update'])))), (!data.active && !data.loading && $.make('div', { className: 'NB-statistics-count' }, "Not active")) ]), + ((data.average_stories_per_month == 0 || data.stories_last_month == 0) && + $.make('div', { className: 'NB-statistics-update-explainer' }, [ + $.make('b', 'Why so infrequently?'), + 'This site has published zero stories in the past month or has averaged less than a single story a month. As soon as it starts publishing at least once a month, it will automatically fetch more frequently.' + ])), (!NEWSBLUR.Globals.is_premium && $.make('div', { className: 'NB-statistics-premium-stats' }, [ $.make('div', { className: 'NB-statistics-update'}, [ $.make('div', { className: 'NB-statistics-label' }, [ diff --git a/node/favicons.coffee b/node/favicons.coffee index 06b0f19a8..6826016f3 100644 --- a/node/favicons.coffee +++ b/node/favicons.coffee @@ -2,7 +2,7 @@ express = require 'express' mongo = require 'mongodb' DEV = process.env.NODE_ENV == 'development' -MONGODB_SERVER = if DEV then 'localhost' else 'db04' +MONGODB_SERVER = if DEV then 'localhost' else 'db24' MONGODB_PORT = parseInt(process.env.MONGODB_PORT or 27017, 10) if DEV diff --git a/node/favicons.js b/node/favicons.js index 55084179a..acb306560 100644 --- a/node/favicons.js +++ b/node/favicons.js @@ -9,7 +9,7 @@ DEV = process.env.NODE_ENV === 'development'; - MONGODB_SERVER = DEV ? 'localhost' : 'db04'; + MONGODB_SERVER = DEV ? 'localhost' : 'db24'; MONGODB_PORT = parseInt(process.env.MONGODB_PORT || 27017, 10); diff --git a/utils/bootstrap_story_hash.py b/utils/bootstrap_story_hash.py new file mode 100644 index 000000000..6c708cd8e --- /dev/null +++ b/utils/bootstrap_story_hash.py @@ -0,0 +1,29 @@ +import time +import pymongo +from django.conf import settings +from apps.rss_feeds.models import MStory, Feed + +db = settings.MONGODB +batch = 0 +start = 0 +for f in xrange(start, Feed.objects.latest('pk').pk): + if f < batch*100000: continue + start = time.time() + try: + cp1 = time.time() - start + # if feed.active_premium_subscribers < 1: continue + stories = MStory.objects.filter(story_feed_id=f, story_hash__exists=False)\ + .only('id', 'story_feed_id', 'story_guid')\ + .read_preference(pymongo.ReadPreference.SECONDARY) + cp2 = time.time() - start + count = 0 + for story in stories: + count += 1 + db.newsblur.stories.update({"_id": story.id}, {"$set": { + "story_hash": story.feed_guid_hash + }}) + cp3 = time.time() - start + print "%s: %3s stories (%s/%s/%s)" % (f, count, round(cp1, 2), round(cp2, 2), round(cp3, 2)) + except Exception, e: + print " ***> (%s) %s" % (f, e) +