mirror of
https://github.com/viq/NewsBlur.git
synced 2025-08-31 22:20:12 +00:00
Merge branch 'master' into circular
* master: Adding task servers to db firewall. Adding redis cache. Returning favicon server. Better munin plugins for redis and mongo. Removing unused redis keys for shared/comments. Dry-run. Adding read preference to migration. Migration to create story_hash for missing feeds. Making a clearer description of slow feeds. Upping feed fetches, slowing down fetch interval for less active feeds, setting premium expire time to one year from most recent payment date (as opposed to borked payment gap logic), and adding logginf or tasking feeds. Switching check for HAProxy'ied socket.io to dev instead of www. Thanks @anaconda! Sync delay in mongodb and updating story hash migration. Upping feed fetches. Fix force close when attempting to fetch a user with a null user.id value Fix force close updating ProfileActivityFragment if no profile response was received Move creating ImageLoader in NewsBlurApplication to onCreate to ensure that base context has been set and prevent force close Boosting fetches. Fix duplicate requirement Conflicts: fabfile.py
This commit is contained in:
Samuel Clay
commit
a67df54101
23 changed files with 388 additions and 141 deletions
|
|
@ -211,14 +211,7 @@ class Profile(models.Model):
|
|||
print " ---> %s payments" % len(payment_history)
|
||||
|
||||
if most_recent_payment_date:
|
||||
payment_gap = 0
|
||||
# If user lapsed and has no gap b/w last payment and expiration,
|
||||
# they only get a full year. Otherwise, give them the gap.
|
||||
if (self.premium_expire and
|
||||
self.premium_expire > datetime.datetime.now() and
|
||||
self.premium_expire > most_recent_payment_date):
|
||||
payment_gap = (self.premium_expire - most_recent_payment_date).days
|
||||
self.premium_expire = most_recent_payment_date + datetime.timedelta(days=365+payment_gap)
|
||||
self.premium_expire = most_recent_payment_date + datetime.timedelta(days=365)
|
||||
self.save()
|
||||
|
||||
def cancel_premium(self):
|
||||
|
|
|
|||
|
|
@ -1,96 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from south.v2 import DataMigration
|
||||
|
||||
class Migration(DataMigration):
|
||||
|
||||
def forwards(self, orm):
|
||||
from apps.rss_feeds.models import MStory, Feed
|
||||
import time
|
||||
|
||||
batch = 0
|
||||
for f in xrange(Feed.objects.latest('pk').pk):
|
||||
if f < batch*100000: continue
|
||||
start = time.time()
|
||||
try:
|
||||
try:
|
||||
feed = Feed.get_by_id(f)
|
||||
except Feed.DoesNotExist:
|
||||
continue
|
||||
if not feed: continue
|
||||
cp1 = time.time() - start
|
||||
if feed.active_premium_subscribers < 1: continue
|
||||
stories = MStory.objects.filter(story_feed_id=feed.pk, story_hash__exists=False)
|
||||
cp2 = time.time() - start
|
||||
for story in stories: story.save()
|
||||
cp3 = time.time() - start
|
||||
print "%3s stories: %s (%s/%s/%s)" % (stories.count(), feed, round(cp1, 2), round(cp2, 2), round(cp3, 2))
|
||||
except Exception, e:
|
||||
print " ***> (%s) %s" % (f, e)
|
||||
|
||||
|
||||
|
||||
def backwards(self, orm):
|
||||
"Write your backwards methods here."
|
||||
|
||||
models = {
|
||||
'rss_feeds.duplicatefeed': {
|
||||
'Meta': {'object_name': 'DuplicateFeed'},
|
||||
'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '764', 'db_index': 'True'}),
|
||||
'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}),
|
||||
'duplicate_link': ('django.db.models.fields.CharField', [], {'max_length': '764', 'null': 'True', 'db_index': 'True'}),
|
||||
'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
|
||||
},
|
||||
'rss_feeds.feed': {
|
||||
'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
|
||||
'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
|
||||
'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
|
||||
'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
|
||||
'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
|
||||
'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
|
||||
'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
|
||||
'errors_since_good': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
|
||||
'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
|
||||
'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '764', 'db_index': 'True'}),
|
||||
'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
|
||||
'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
|
||||
'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
|
||||
'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
|
||||
'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
|
||||
'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
|
||||
'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
|
||||
'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
|
||||
'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
|
||||
'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
|
||||
'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
|
||||
'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
|
||||
's3_icon': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
|
||||
's3_page': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
|
||||
'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
|
||||
},
|
||||
'rss_feeds.feeddata': {
|
||||
'Meta': {'object_name': 'FeedData'},
|
||||
'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
|
||||
'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
|
||||
'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
|
||||
'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
|
||||
}
|
||||
}
|
||||
|
||||
complete_apps = ['rss_feeds']
|
||||
symmetrical = True
|
||||
|
|
@ -1222,9 +1222,11 @@ class Feed(models.Model):
|
|||
# 1 update per day = 1 hours
|
||||
# 10 updates = 20 minutes
|
||||
updates_per_day_delay = 3 * 60 / max(.25, ((max(0, self.active_subscribers)**.2)
|
||||
* (updates_per_month**0.35)))
|
||||
if self.premium_subscribers > 0:
|
||||
updates_per_day_delay /= min(self.active_subscribers+self.premium_subscribers, 5)
|
||||
* (updates_per_month**0.25)))
|
||||
if self.active_premium_subscribers > 0:
|
||||
updates_per_day_delay /= min(self.active_subscribers+self.active_premium_subscribers, 4)
|
||||
updates_per_day_delay = int(updates_per_day_delay)
|
||||
|
||||
# Lots of subscribers = lots of updates
|
||||
# 24 hours for 0 subscribers.
|
||||
# 4 hours for 1 subscriber.
|
||||
|
|
@ -1234,6 +1236,7 @@ class Feed(models.Model):
|
|||
subscriber_bonus = 12 * 60 / max(.167, max(0, self.active_subscribers)**3)
|
||||
if self.premium_subscribers > 0:
|
||||
subscriber_bonus /= min(self.active_subscribers+self.premium_subscribers, 5)
|
||||
subscriber_bonus = int(subscriber_bonus)
|
||||
|
||||
slow_punishment = 0
|
||||
if self.num_subscribers <= 1:
|
||||
|
|
@ -1259,16 +1262,21 @@ class Feed(models.Model):
|
|||
total = 60*24*30
|
||||
|
||||
if verbose:
|
||||
print "[%s] %s (%s/%s/%s/%s), %s, %s: %s" % (self, updates_per_day_delay,
|
||||
self.num_subscribers, self.active_subscribers,
|
||||
self.premium_subscribers, self.active_premium_subscribers,
|
||||
subscriber_bonus, slow_punishment, total)
|
||||
logging.debug(" ---> [%-30s] Fetched every %sm (%s+%s+%s) Subs: %s/%s Stories: %s/%s" % (
|
||||
unicode(self)[:30], total,
|
||||
updates_per_day_delay,
|
||||
subscriber_bonus,
|
||||
slow_punishment,
|
||||
self.num_subscribers,
|
||||
self.active_premium_subscribers,
|
||||
self.average_stories_per_month,
|
||||
self.stories_last_month))
|
||||
random_factor = random.randint(0, total) / 4
|
||||
|
||||
return total, random_factor*8
|
||||
|
||||
def set_next_scheduled_update(self, verbose=True):
|
||||
total, random_factor = self.get_next_scheduled_update(force=True, verbose=False)
|
||||
def set_next_scheduled_update(self, verbose=False, skip_scheduling=False):
|
||||
total, random_factor = self.get_next_scheduled_update(force=True, verbose=verbose)
|
||||
|
||||
if self.errors_since_good:
|
||||
total = total * self.errors_since_good
|
||||
|
|
@ -1281,7 +1289,8 @@ class Feed(models.Model):
|
|||
minutes = total + random_factor)
|
||||
|
||||
self.min_to_decay = total
|
||||
self.next_scheduled_update = next_scheduled_update
|
||||
if not skip_scheduling:
|
||||
self.next_scheduled_update = next_scheduled_update
|
||||
|
||||
self.save()
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ class TaskFeeds(Task):
|
|||
from apps.rss_feeds.models import Feed
|
||||
settings.LOG_TO_STREAM = True
|
||||
now = datetime.datetime.utcnow()
|
||||
start = time.time()
|
||||
|
||||
# Active feeds
|
||||
popular_feeds = Feed.objects.filter(
|
||||
|
|
@ -28,7 +29,7 @@ class TaskFeeds(Task):
|
|||
next_scheduled_update__lte=now,
|
||||
active=True,
|
||||
active_subscribers__gte=1
|
||||
).order_by('?')[:200]
|
||||
).order_by('?')[:600]
|
||||
active_count = feeds.count()
|
||||
|
||||
# Force refresh feeds
|
||||
|
|
@ -71,7 +72,9 @@ class TaskFeeds(Task):
|
|||
Feed.task_feeds(refresh_feeds, verbose=False)
|
||||
if inactive_feeds: Feed.task_feeds(inactive_feeds, verbose=False)
|
||||
if old_feeds: Feed.task_feeds(old_feeds, verbose=False)
|
||||
|
||||
|
||||
logging.debug(" ---> ~FBTasking took %.2s seconds" % (time.time() - start))
|
||||
|
||||
|
||||
class UpdateFeeds(Task):
|
||||
name = 'update-feeds'
|
||||
|
|
|
|||
|
|
@ -134,6 +134,8 @@ def load_feed_statistics(request, feed_id):
|
|||
user = get_user(request)
|
||||
stats = dict()
|
||||
feed = get_object_or_404(Feed, pk=feed_id)
|
||||
feed.count_subscribers()
|
||||
feed.set_next_scheduled_update(verbose=True, skip_scheduling=True)
|
||||
feed.save_feed_story_history_statistics()
|
||||
feed.save_classifier_counts()
|
||||
|
||||
|
|
|
|||
27
apps/social/migrations/0007_prune_redis_hash.py
Normal file
27
apps/social/migrations/0007_prune_redis_hash.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import datetime
|
||||
from south.db import db
|
||||
from south.v2 import DataMigration
|
||||
from django.db import models
|
||||
import redis
|
||||
from django.conf import settings
|
||||
|
||||
class Migration(DataMigration):
|
||||
|
||||
def forwards(self, orm):
|
||||
r = redis.Redis(connection_pool=settings.REDIS_POOL)
|
||||
keys = r.keys("*:*:????????????????????????????????????????")
|
||||
print " ---> %s keys" % len(keys)
|
||||
for key in keys:
|
||||
print "Deleting %s" % key
|
||||
r.delete(key)
|
||||
|
||||
def backwards(self, orm):
|
||||
"Write your backwards methods here."
|
||||
|
||||
models = {
|
||||
|
||||
}
|
||||
|
||||
complete_apps = ['social']
|
||||
symmetrical = True
|
||||
|
|
@ -16,6 +16,8 @@ logappend=true
|
|||
|
||||
slowms=100
|
||||
|
||||
syncdelay=5
|
||||
|
||||
rest = true
|
||||
#profile = 2
|
||||
# Enables periodic logging of CPU utilization and I/O wait
|
||||
|
|
|
|||
94
config/munin/mongo_indexsize
Normal file
94
config/munin/mongo_indexsize
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# vim: set sts=4 sw=4 encoding=utf-8
|
||||
|
||||
# Copyright (c) 2010, Rene Jochum
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
# * Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
# * Neither the name of the Rene Jochum nor the
|
||||
# names of its contributors may be used to endorse or promote products
|
||||
# derived from this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
|
||||
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
from pymongo import Connection
|
||||
import os
|
||||
|
||||
settings_host = os.environ.get("host", "127.0.0.1")
|
||||
settings_port = 27017
|
||||
settings_db = 'newsblur'
|
||||
|
||||
def getCollstats():
|
||||
global settings_host, settings_port, settings_db, settings_user, settings_password
|
||||
print locals(), settings_host, settings_port
|
||||
if settings_user and settings_password:
|
||||
settings_host = "%s:%s@%s" % (settings_user, settings_password, settings_host)
|
||||
con = Connection(settings_host, int(settings_port), slave_okay=True)
|
||||
db = con[settings_db]
|
||||
|
||||
for coll in db.collection_names():
|
||||
if coll.startswith('system.'):
|
||||
continue
|
||||
stats = db.command("collstats", coll)
|
||||
yield ("%s_size" % coll.replace('.', '_'), long(stats['totalIndexSize']),)
|
||||
|
||||
con.disconnect()
|
||||
|
||||
|
||||
def doData():
|
||||
for coll, stats in getCollstats():
|
||||
print "%s.value %s" % (coll, stats)
|
||||
|
||||
|
||||
def doConfig():
|
||||
|
||||
print "graph_title MongoDB collection index sizes"
|
||||
print "graph_args --base 1024 -l 0"
|
||||
print "graph_vlabel Kb"
|
||||
print "graph_category MongoDB"
|
||||
print "graph_total total"
|
||||
|
||||
for k,v in getCollstats():
|
||||
print "%s.label %s" % (k, k)
|
||||
print "%s.min 0" % k
|
||||
print "%s.draw LINE1" % k
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from sys import argv
|
||||
from os import environ
|
||||
|
||||
# Could be done by a for loop
|
||||
# but i think if's are faster
|
||||
if 'HOST' in environ:
|
||||
settings_host = environ['HOST']
|
||||
if 'PORT' in environ:
|
||||
settings_port = environ['PORT']
|
||||
if 'DB' in environ:
|
||||
settings_db = environ['DB']
|
||||
if 'user' in environ:
|
||||
settings_user = environ['user']
|
||||
if 'password' in environ:
|
||||
settings_password = environ['password']
|
||||
print locals()
|
||||
if len(argv) > 1 and argv[1] == "config":
|
||||
doConfig()
|
||||
else:
|
||||
doData()
|
||||
|
|
@ -18,8 +18,8 @@ class NBMuninGraph(MuninGraph):
|
|||
}
|
||||
|
||||
stats = self.stats
|
||||
graph.update(dict((("%s.label" % s['_id'], s['_id']) for s in stats)))
|
||||
graph.update(dict((("%s.draw" % s['_id'], 'LINE1') for s in stats)))
|
||||
graph.update(dict((("%s.label" % s, s) for s in stats.keys())))
|
||||
graph.update(dict((("%s.draw" % s, 'LINE1') for s in stats.keys())))
|
||||
|
||||
return graph
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,15 @@ upstream app_server {
|
|||
server 127.0.0.1:8000 fail_timeout=10 max_fails=3 ;
|
||||
}
|
||||
|
||||
upstream icon_server {
|
||||
server 127.0.0.1:3030 fail_timeout=2 max_fails=3;
|
||||
server 127.0.0.1:8000 backup;
|
||||
}
|
||||
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
# listen 443 default_server ssl;
|
||||
listen 443 default_server ssl;
|
||||
|
||||
# ssl on;
|
||||
ssl_certificate /srv/newsblur/config/certificates/newsblur.com.crt;
|
||||
|
|
@ -80,8 +86,15 @@ server {
|
|||
alias /srv/newsblur/media/robots.txt;
|
||||
}
|
||||
|
||||
location /munin/static/ {
|
||||
alias /etc/munin/static/;
|
||||
}
|
||||
|
||||
location /munin/ {
|
||||
alias /var/cache/munin/www/;
|
||||
fastcgi_split_path_info ^(/munin)(.*);
|
||||
fastcgi_param PATH_INFO $fastcgi_path_info;
|
||||
fastcgi_pass unix:/var/run/munin/fcgi-html.sock;
|
||||
include fastcgi_params;
|
||||
}
|
||||
|
||||
location ^~ /cgi-bin/munin-cgi-graph/ {
|
||||
|
|
@ -92,7 +105,12 @@ server {
|
|||
}
|
||||
|
||||
location ^~ /rss_feeds/icon/ {
|
||||
rewrite ^(.*)$ /media/img/icons/silk/world.png;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_redirect off;
|
||||
|
||||
proxy_pass http://icon_server;
|
||||
}
|
||||
|
||||
location / {
|
||||
|
|
|
|||
|
|
@ -9,10 +9,10 @@ django-compress==1.0.1
|
|||
django-extensions==1.1.1
|
||||
django-mailgun==0.2.1
|
||||
django-redis-sessions==0.3.1
|
||||
django-redis-cache==0.9.7
|
||||
django-ses==0.4.1
|
||||
django-subdomains==2.0.3
|
||||
Django>=1.5,<1.6
|
||||
django_ses
|
||||
Fabric==1.6.0
|
||||
gunicorn==0.17.2
|
||||
httplib2==0.8
|
||||
|
|
@ -34,6 +34,7 @@ PyYAML==3.10
|
|||
raven==3.1.17
|
||||
readline==6.2.4.1
|
||||
redis==2.7.2
|
||||
hiredis==0.1.1
|
||||
requests==1.1.0
|
||||
seacucumber==1.5
|
||||
South==0.7.6
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ NAME=spawn-fcgi-munin-graph
|
|||
PID_FILE=/var/run/munin/fcgi-graph.pid
|
||||
SOCK_FILE=/var/run/munin/fcgi-graph.sock
|
||||
DAEMON=/usr/bin/spawn-fcgi
|
||||
DAEMON_OPTS="-s $SOCK_FILE -U www-data -u www-data -g www-data /usr/lib/cgi-bin/munin-cgi-graph -P $PID_FILE"
|
||||
DAEMON_OPTS="-s $SOCK_FILE -U nginx -u nginx -g www-data /usr/lib/cgi-bin/munin-cgi-graph -P $PID_FILE"
|
||||
|
||||
# --------------------------------------------------------------
|
||||
# No edits necessary beyond this line
|
||||
|
|
|
|||
119
config/spawn_fcgi_munin_html.conf
Normal file
119
config/spawn_fcgi_munin_html.conf
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
#! /bin/sh
|
||||
|
||||
### BEGIN INIT INFO
|
||||
# Provides: spawn-fcgi-munin-html
|
||||
# Required-Start: $all
|
||||
# Required-Stop: $all
|
||||
# Default-Start: 2 3 4 5
|
||||
# Default-Stop: 0 1 6
|
||||
# Description: starts FastCGI for Munin-Html
|
||||
### END INIT INFO
|
||||
# --------------------------------------------------------------
|
||||
# Munin-CGI-HTML Spawn-FCGI Startscript by Julien Schmidt
|
||||
# eMail: munin-trac at julienschmidt.com
|
||||
# www: http://www.julienschmidt.com
|
||||
# --------------------------------------------------------------
|
||||
# Install:
|
||||
# 1. Copy this file to /etc/init.d
|
||||
# 2. Edit the variables below
|
||||
# 3. run "update-rc.d spawn-fcgi-munin-html defaults"
|
||||
# --------------------------------------------------------------
|
||||
# Last Update: 13. November 2012
|
||||
#
|
||||
# Please change the following variables:
|
||||
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
||||
NAME=spawn-fcgi-munin-html
|
||||
PID_FILE=/var/run/munin/fcgi-html.pid
|
||||
SOCK_FILE=/var/run/munin/fcgi-html.sock
|
||||
DAEMON=/usr/bin/spawn-fcgi
|
||||
DAEMON_OPTS="-s $SOCK_FILE -U nginx -u nginx -g www-data /usr/lib/cgi-bin/munin-cgi-html -P $PID_FILE"
|
||||
|
||||
# --------------------------------------------------------------
|
||||
# No edits necessary beyond this line
|
||||
# --------------------------------------------------------------
|
||||
|
||||
if [ ! -x $DAEMON ]; then
|
||||
echo "File not found or is not executable: $DAEMON!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
status() {
|
||||
if [ ! -r $PID_FILE ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
FCGI_PID=`cat $PID_FILE`
|
||||
if [ -z "${FCGI_PID}" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
FCGI_RUNNING=`ps -p ${FCGI_PID} | grep ${FCGI_PID}`
|
||||
if [ -z "${FCGI_RUNNING}" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
start() {
|
||||
if status; then
|
||||
echo "FCGI is already running!"
|
||||
exit 1
|
||||
else
|
||||
$DAEMON $DAEMON_OPTS
|
||||
fi
|
||||
}
|
||||
|
||||
stop () {
|
||||
if ! status; then
|
||||
echo "No PID-file at $PID_FILE found or PID not valid. Maybe not running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Kill process
|
||||
kill -9 `cat $PID_FILE`
|
||||
|
||||
# Remove PID-file
|
||||
rm -f $PID_FILE
|
||||
|
||||
# Remove Sock-File
|
||||
rm -f $SOCK_FILE
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
start)
|
||||
echo "Starting $NAME: "
|
||||
start
|
||||
echo "... DONE"
|
||||
;;
|
||||
|
||||
stop)
|
||||
echo "Stopping $NAME: "
|
||||
stop
|
||||
echo "... DONE"
|
||||
;;
|
||||
|
||||
force-reload|restart)
|
||||
echo "Stopping $NAME: "
|
||||
stop
|
||||
echo "Starting $NAME: "
|
||||
start
|
||||
echo "... DONE"
|
||||
;;
|
||||
|
||||
status)
|
||||
if status; then
|
||||
echo "FCGI is RUNNING"
|
||||
else
|
||||
echo "FCGI is NOT RUNNING"
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: $0 {start|stop|restart|status}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
exit 0
|
||||
24
fabfile.py
vendored
24
fabfile.py
vendored
|
|
@ -154,7 +154,7 @@ def deploy_node():
|
|||
with cd(env.NEWSBLUR_PATH):
|
||||
run('sudo supervisorctl restart node_unread')
|
||||
run('sudo supervisorctl restart node_unread_ssl')
|
||||
# run('sudo supervisorctl restart node_favicons')
|
||||
run('sudo supervisorctl restart node_favicons')
|
||||
|
||||
def gunicorn_restart():
|
||||
restart_gunicorn()
|
||||
|
|
@ -637,7 +637,7 @@ def configure_node():
|
|||
sudo('rm -fr /etc/supervisor/conf.d/node.conf')
|
||||
put('config/supervisor_node_unread.conf', '/etc/supervisor/conf.d/node_unread.conf', use_sudo=True)
|
||||
put('config/supervisor_node_unread_ssl.conf', '/etc/supervisor/conf.d/node_unread_ssl.conf', use_sudo=True)
|
||||
# put('config/supervisor_node_favicons.conf', '/etc/supervisor/conf.d/node_favicons.conf', use_sudo=True)
|
||||
put('config/supervisor_node_favicons.conf', '/etc/supervisor/conf.d/node_favicons.conf', use_sudo=True)
|
||||
sudo('supervisorctl reload')
|
||||
|
||||
@parallel
|
||||
|
|
@ -737,7 +737,8 @@ def setup_db_firewall():
|
|||
for ip in set(env.roledefs['app'] +
|
||||
env.roledefs['dbdo'] +
|
||||
env.roledefs['dev'] +
|
||||
env.roledefs['debug']):
|
||||
env.roledefs['debug'] +
|
||||
env.roledefs['task']):
|
||||
sudo('ufw allow proto tcp from %s to any port %s' % (
|
||||
ip,
|
||||
','.join(map(str, ports))
|
||||
|
|
@ -824,16 +825,29 @@ def setup_redis():
|
|||
sudo('/etc/init.d/redis start')
|
||||
|
||||
def setup_munin():
|
||||
sudo('apt-get update')
|
||||
# sudo('apt-get update')
|
||||
sudo('apt-get install -y munin munin-node munin-plugins-extra spawn-fcgi')
|
||||
put('config/munin.conf', '/etc/munin/munin.conf', use_sudo=True)
|
||||
put('config/spawn_fcgi_munin_graph.conf', '/etc/init.d/spawn_fcgi_munin_graph', use_sudo=True)
|
||||
put('config/spawn_fcgi_munin_html.conf', '/etc/init.d/spawn_fcgi_munin_html', use_sudo=True)
|
||||
sudo('chmod u+x /etc/init.d/spawn_fcgi_munin_graph')
|
||||
sudo('chmod u+x /etc/init.d/spawn_fcgi_munin_html')
|
||||
with settings(warn_only=True):
|
||||
sudo('chown nginx.www-data munin-cgi*')
|
||||
with settings(warn_only=True):
|
||||
sudo('/etc/init.d/spawn_fcgi_munin_graph stop')
|
||||
sudo('/etc/init.d/spawn_fcgi_munin_graph start')
|
||||
sudo('update-rc.d spawn_fcgi_munin_graph defaults')
|
||||
sudo('/etc/init.d/spawn_fcgi_munin_html stop')
|
||||
sudo('/etc/init.d/spawn_fcgi_munin_html start')
|
||||
sudo('update-rc.d spawn_fcgi_munin_html defaults')
|
||||
sudo('/etc/init.d/munin-node restart')
|
||||
|
||||
with settings(warn_only=True):
|
||||
sudo('chown nginx.www-data munin-cgi*')
|
||||
with settings(warn_only=True):
|
||||
sudo('/etc/init.d/spawn_fcgi_munin_graph start')
|
||||
sudo('/etc/init.d/spawn_fcgi_munin_html start')
|
||||
|
||||
|
||||
def setup_db_munin():
|
||||
sudo('cp -frs %s/config/munin/mongo* /etc/munin/plugins/' % env.NEWSBLUR_PATH)
|
||||
|
|
|
|||
|
|
@ -23,10 +23,21 @@ DEBUG_ASSETS = DEBUG
|
|||
MEDIA_URL = '/media/'
|
||||
SECRET_KEY = 'YOUR SECRET KEY'
|
||||
|
||||
CACHE_BACKEND = 'dummy:///'
|
||||
# CACHE_BACKEND = 'dummy:///'
|
||||
# CACHE_BACKEND = 'locmem:///'
|
||||
# CACHE_BACKEND = 'memcached://127.0.0.1:11211'
|
||||
|
||||
CACHES = {
|
||||
'default': {
|
||||
'BACKEND': 'redis_cache.RedisCache',
|
||||
'LOCATION': '127.0.0.1:6379',
|
||||
'OPTIONS': {
|
||||
'DB': 6,
|
||||
'PARSER_CLASS': 'redis.connection.HiredisParser'
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
|
||||
|
||||
# Set this to the username that is shown on the homepage to unauthenticated users.
|
||||
|
|
|
|||
|
|
@ -8,11 +8,12 @@ public class NewsBlurApplication extends Application {
|
|||
|
||||
ImageLoader imageLoader;
|
||||
|
||||
public NewsBlurApplication() {
|
||||
super();
|
||||
@Override
|
||||
public void onCreate() {
|
||||
super.onCreate();
|
||||
imageLoader = new ImageLoader(this);
|
||||
}
|
||||
|
||||
|
||||
public ImageLoader getImageLoader() {
|
||||
return imageLoader;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,9 +91,13 @@ public class Profile extends SherlockFragmentActivity {
|
|||
} else {
|
||||
apiManager.updateUserProfile();
|
||||
user = PrefsUtils.getUserDetails(Profile.this);
|
||||
profileResponse = apiManager.getUser(user.id);
|
||||
if (profileResponse != null) {
|
||||
activities = profileResponse.activities;
|
||||
// check user.id has been set. If previous attempts to update the user details
|
||||
// have failed then user.id == null would cause a force close
|
||||
if (user.id != null) {
|
||||
profileResponse = apiManager.getUser(user.id);
|
||||
if (profileResponse != null) {
|
||||
activities = profileResponse.activities;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
|
|
@ -103,8 +107,10 @@ public class Profile extends SherlockFragmentActivity {
|
|||
protected void onPostExecute(Void result) {
|
||||
if (user != null && detailsFragment != null && activitiesFragment != null) {
|
||||
detailsFragment.setUser(Profile.this, user, TextUtils.isEmpty(userId));
|
||||
// TODO still sometimes causes a force close - is activities null ?
|
||||
activitiesFragment.setActivitiesAndUser(Profile.this, activities, user);
|
||||
// activities could be null if no profile response was received
|
||||
if (activities != null) {
|
||||
activitiesFragment.setActivitiesAndUser(Profile.this, activities, user);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6142,6 +6142,15 @@ form.opml_import_form input {
|
|||
overflow: hidden;
|
||||
margin: 0 0 12px 0;
|
||||
}
|
||||
.NB-modal-statistics .NB-statistics-update-explainer {
|
||||
clear: both;
|
||||
margin: 12px 24px 0;
|
||||
font-size: 10px;
|
||||
color: #808080;
|
||||
}
|
||||
.NB-modal-statistics .NB-statistics-update-explainer b {
|
||||
padding-right: 8px;
|
||||
}
|
||||
.NB-modal-statistics .NB-statistics-premium-stats {
|
||||
border-top: 1px solid #E0E0E0;
|
||||
padding: 12px 0 0;
|
||||
|
|
|
|||
|
|
@ -3944,9 +3944,9 @@
|
|||
} else if (force || !this.socket || !this.socket.socket.connected) {
|
||||
var server = window.location.protocol + '//' + window.location.hostname;
|
||||
var https = _.string.startsWith(window.location.protocol, 'https');
|
||||
var www = _.string.contains(window.location.href, 'www.newsblur.com');
|
||||
var dev = _.string.contains(window.location.href, 'dev.newsblur.com');
|
||||
var port = https ? 443 : 80;
|
||||
if (NEWSBLUR.Globals.debug || !www) {
|
||||
if (NEWSBLUR.Globals.debug || dev) {
|
||||
port = https ? 8889 : 8888;
|
||||
}
|
||||
this.socket = this.socket || io.connect(server, {
|
||||
|
|
|
|||
|
|
@ -103,6 +103,11 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, {
|
|||
(data.active && $.make('div', { className: 'NB-statistics-count' }, ' ' + (data['next_update'] && ('in ' + data['next_update'])))),
|
||||
(!data.active && !data.loading && $.make('div', { className: 'NB-statistics-count' }, "Not active"))
|
||||
]),
|
||||
((data.average_stories_per_month == 0 || data.stories_last_month == 0) &&
|
||||
$.make('div', { className: 'NB-statistics-update-explainer' }, [
|
||||
$.make('b', 'Why so infrequently?'),
|
||||
'This site has published zero stories in the past month or has averaged less than a single story a month. As soon as it starts publishing at least once a month, it will automatically fetch more frequently.'
|
||||
])),
|
||||
(!NEWSBLUR.Globals.is_premium && $.make('div', { className: 'NB-statistics-premium-stats' }, [
|
||||
$.make('div', { className: 'NB-statistics-update'}, [
|
||||
$.make('div', { className: 'NB-statistics-label' }, [
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ express = require 'express'
|
|||
mongo = require 'mongodb'
|
||||
|
||||
DEV = process.env.NODE_ENV == 'development'
|
||||
MONGODB_SERVER = if DEV then 'localhost' else 'db04'
|
||||
MONGODB_SERVER = if DEV then 'localhost' else 'db24'
|
||||
MONGODB_PORT = parseInt(process.env.MONGODB_PORT or 27017, 10)
|
||||
|
||||
if DEV
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
DEV = process.env.NODE_ENV === 'development';
|
||||
|
||||
MONGODB_SERVER = DEV ? 'localhost' : 'db04';
|
||||
MONGODB_SERVER = DEV ? 'localhost' : 'db24';
|
||||
|
||||
MONGODB_PORT = parseInt(process.env.MONGODB_PORT || 27017, 10);
|
||||
|
||||
|
|
|
|||
29
utils/bootstrap_story_hash.py
Normal file
29
utils/bootstrap_story_hash.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import time
|
||||
import pymongo
|
||||
from django.conf import settings
|
||||
from apps.rss_feeds.models import MStory, Feed
|
||||
|
||||
db = settings.MONGODB
|
||||
batch = 0
|
||||
start = 0
|
||||
for f in xrange(start, Feed.objects.latest('pk').pk):
|
||||
if f < batch*100000: continue
|
||||
start = time.time()
|
||||
try:
|
||||
cp1 = time.time() - start
|
||||
# if feed.active_premium_subscribers < 1: continue
|
||||
stories = MStory.objects.filter(story_feed_id=f, story_hash__exists=False)\
|
||||
.only('id', 'story_feed_id', 'story_guid')\
|
||||
.read_preference(pymongo.ReadPreference.SECONDARY)
|
||||
cp2 = time.time() - start
|
||||
count = 0
|
||||
for story in stories:
|
||||
count += 1
|
||||
db.newsblur.stories.update({"_id": story.id}, {"$set": {
|
||||
"story_hash": story.feed_guid_hash
|
||||
}})
|
||||
cp3 = time.time() - start
|
||||
print "%s: %3s stories (%s/%s/%s)" % (f, count, round(cp1, 2), round(cp2, 2), round(cp3, 2))
|
||||
except Exception, e:
|
||||
print " ***> (%s) %s" % (f, e)
|
||||
|
||||
Loading…
Add table
Reference in a new issue