mirror of
https://github.com/samuelclay/NewsBlur.git
synced 2025-08-05 16:58:59 +00:00
b5efad9142
* master: (66 commits) Too many stories? Don't animate transitions when switching intelligence levels. Fixing typo in iphone app that preventing mark folder as read when the visible stories option was showing. Updating the mongo db copy util to also look for updated stories. Globals.is_staff on user, not profile. Make the stats referesh every minute instead of every 10 minutes for staff. Fixing offsets in river of news. Adding ciphering to usernames in log, so I can finally take screenshots of the most colorful logs of all time. Adding my very special mongo serialization backup utility to get around the damn unrepairable mongo database. This is taking 14 hours to run. Auto-refreshing feedback (1 min for staff, 10 min for everybody). Fixing exception around multiple feeds during Google Reader import process. Also switching rate limit to status code 429. Adding Nokia MeeGo client to user agents. Only show raw feeds in feed autocomplete. Handling iphone version for new users on ios app. FIXING THE WORST BUG OF MY LIFE -- finally figured out what was causing the story-shows-as-unread bug. Also fixed enclosures on certain types of feeds. Fixing menu manage open position to compensate for additional menu items. Reducing the amount of work done by feed fetching when there are no new stories. Fixing emergency bug around trimming feeds where the cursor is changing. Dammit mongo. Simplifying ufw ports in fabfile. Adding env.user. Launching the iPhone app on the front-page. Big deal. Minor cleanup of river stories view. Cleaning up mongoengine imports and settings for default MongoDB. ... Conflicts: local_settings.py.template
83 lines
3.1 KiB
Python
83 lines
3.1 KiB
Python
from django.core.management.base import BaseCommand
|
|
from django.conf import settings
|
|
from django.contrib.auth.models import User
|
|
from apps.rss_feeds.models import Feed
|
|
from optparse import make_option
|
|
from utils import feed_fetcher
|
|
from utils.management_functions import daemonize
|
|
import django
|
|
import socket
|
|
import datetime
|
|
import redis
|
|
|
|
|
|
class Command(BaseCommand):
|
|
option_list = BaseCommand.option_list + (
|
|
make_option("-f", "--feed", default=None),
|
|
make_option("-d", "--daemon", dest="daemonize", action="store_true"),
|
|
make_option("-F", "--force", dest="force", action="store_true"),
|
|
make_option("-s", "--single_threaded", dest="single_threaded", action="store_true"),
|
|
make_option('-t', '--timeout', type='int', default=10,
|
|
help='Wait timeout in seconds when connecting to feeds.'),
|
|
make_option('-u', '--username', type='str', dest='username'),
|
|
make_option('-V', '--verbose', action='store_true',
|
|
dest='verbose', default=False, help='Verbose output.'),
|
|
make_option('-S', '--skip', type='int',
|
|
dest='skip', default=0, help='Skip stories per month < #.'),
|
|
make_option('-w', '--workerthreads', type='int', default=4,
|
|
help='Worker threads that will fetch feeds in parallel.'),
|
|
)
|
|
|
|
def handle(self, *args, **options):
|
|
if options['daemonize']:
|
|
daemonize()
|
|
|
|
settings.LOG_TO_STREAM = True
|
|
now = datetime.datetime.utcnow()
|
|
|
|
if options['skip']:
|
|
feeds = Feed.objects.filter(next_scheduled_update__lte=now,
|
|
average_stories_per_month__lt=options['skip'],
|
|
active=True)
|
|
print " ---> Skipping %s feeds" % feeds.count()
|
|
for feed in feeds:
|
|
feed.set_next_scheduled_update()
|
|
print '.',
|
|
return
|
|
|
|
socket.setdefaulttimeout(options['timeout'])
|
|
if options['force']:
|
|
feeds = Feed.objects.all()
|
|
elif options['username']:
|
|
feeds = Feed.objects.filter(subscribers__user=User.objects.get(username=options['username']))
|
|
else:
|
|
feeds = Feed.objects.filter(next_scheduled_update__lte=now, active=True)
|
|
|
|
feeds = feeds.order_by('?')
|
|
|
|
for f in feeds:
|
|
f.queued_date = datetime.datetime.utcnow()
|
|
f.set_next_scheduled_update()
|
|
|
|
num_workers = min(len(feeds), options['workerthreads'])
|
|
if options['single_threaded']:
|
|
num_workers = 1
|
|
|
|
options['compute_scores'] = True
|
|
|
|
disp = feed_fetcher.Dispatcher(options, num_workers)
|
|
|
|
feeds_queue = []
|
|
for _ in range(num_workers):
|
|
feeds_queue.append([])
|
|
|
|
i = 0
|
|
for feed in feeds:
|
|
feeds_queue[i%num_workers].append(feed.pk)
|
|
i += 1
|
|
disp.add_jobs(feeds_queue, i)
|
|
|
|
django.db.connection.close()
|
|
|
|
print " ---> Fetching %s feeds..." % feeds.count()
|
|
disp.run_jobs()
|