mirror of
https://github.com/viq/NewsBlur.git
synced 2025-04-13 09:38:09 +00:00
88 lines
3.5 KiB
Python
88 lines
3.5 KiB
Python
from django.core.management.base import BaseCommand
|
|
from django.conf import settings
|
|
from django.contrib.auth.models import User
|
|
from apps.statistics.models import MStatistics
|
|
from apps.rss_feeds.models import Feed
|
|
from apps.reader.models import UserSubscription
|
|
from optparse import make_option
|
|
from utils import feed_fetcher
|
|
from utils.management_functions import daemonize
|
|
import django
|
|
import socket
|
|
import datetime
|
|
|
|
|
|
class Command(BaseCommand):
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument("-f", "--feed", default=None)
|
|
parser.add_argument("-d", "--daemon", dest="daemonize", action="store_true")
|
|
parser.add_argument("-F", "--force", dest="force", action="store_true")
|
|
parser.add_argument("-s", "--single_threaded", dest="single_threaded", action="store_true")
|
|
parser.add_argument('-t', '--timeout', type=int, default=10,
|
|
help='Wait timeout in seconds when connecting to feeds.')
|
|
parser.add_argument('-u', '--username', type=str, dest='username')
|
|
parser.add_argument('-V', '--verbose', action='store_true',
|
|
dest='verbose', default=False, help='Verbose output.')
|
|
parser.add_argument('-S', '--skip', type=int,
|
|
dest='skip', default=0, help='Skip stories per month < #.')
|
|
parser.add_argument('-w', '--workerthreads', type=int, default=4,
|
|
help='Worker threads that will fetch feeds in parallel.')
|
|
|
|
def handle(self, *args, **options):
|
|
if options['daemonize']:
|
|
daemonize()
|
|
|
|
settings.LOG_TO_STREAM = True
|
|
now = datetime.datetime.utcnow()
|
|
|
|
if options['skip']:
|
|
feeds = Feed.objects.filter(next_scheduled_update__lte=now,
|
|
average_stories_per_month__lt=options['skip'],
|
|
active=True)
|
|
print(" ---> Skipping %s feeds" % feeds.count())
|
|
for feed in feeds:
|
|
feed.set_next_scheduled_update()
|
|
print('.', end=' ')
|
|
return
|
|
|
|
socket.setdefaulttimeout(options['timeout'])
|
|
if options['force']:
|
|
feeds = Feed.objects.all()
|
|
elif options['username']:
|
|
usersubs = UserSubscription.objects.filter(user=User.objects.get(username=options['username']), active=True)
|
|
feeds = Feed.objects.filter(pk__in=usersubs.values('feed_id'))
|
|
elif options['feed']:
|
|
feeds = Feed.objects.filter(pk=options['feed'])
|
|
else:
|
|
feeds = Feed.objects.filter(next_scheduled_update__lte=now, active=True)
|
|
|
|
feeds = feeds.order_by('?')
|
|
|
|
for f in feeds:
|
|
f.set_next_scheduled_update()
|
|
|
|
num_workers = min(len(feeds), options['workerthreads'])
|
|
if options['single_threaded']:
|
|
num_workers = 1
|
|
|
|
options['compute_scores'] = True
|
|
options['quick'] = float(MStatistics.get('quick_fetch', 0))
|
|
options['updates_off'] = MStatistics.get('updates_off', False)
|
|
|
|
disp = feed_fetcher.Dispatcher(options, num_workers)
|
|
|
|
feeds_queue = []
|
|
for _ in range(num_workers):
|
|
feeds_queue.append([])
|
|
|
|
i = 0
|
|
for feed in feeds:
|
|
feeds_queue[i%num_workers].append(feed.pk)
|
|
i += 1
|
|
disp.add_jobs(feeds_queue, i)
|
|
|
|
django.db.connection.close()
|
|
|
|
print(" ---> Fetching %s feeds..." % feeds.count())
|
|
disp.run_jobs()
|