mirror of
https://github.com/viq/NewsBlur.git
synced 2025-04-13 09:38:09 +00:00
99 lines
3.4 KiB
Python
99 lines
3.4 KiB
Python
import datetime
|
|
import socket
|
|
from optparse import make_option
|
|
|
|
import django
|
|
from django.conf import settings
|
|
from django.contrib.auth.models import User
|
|
from django.core.management.base import BaseCommand
|
|
|
|
from apps.reader.models import UserSubscription
|
|
from apps.rss_feeds.models import Feed
|
|
from apps.statistics.models import MStatistics
|
|
from utils import feed_fetcher
|
|
from utils.management_functions import daemonize
|
|
|
|
|
|
class Command(BaseCommand):
|
|
def add_arguments(self, parser):
|
|
parser.add_argument("-f", "--feed", default=None)
|
|
parser.add_argument("-d", "--daemon", dest="daemonize", action="store_true")
|
|
parser.add_argument("-F", "--force", dest="force", action="store_true")
|
|
parser.add_argument("-s", "--single_threaded", dest="single_threaded", action="store_true")
|
|
parser.add_argument(
|
|
"-t", "--timeout", type=int, default=10, help="Wait timeout in seconds when connecting to feeds."
|
|
)
|
|
parser.add_argument("-u", "--username", type=str, dest="username")
|
|
parser.add_argument(
|
|
"-V", "--verbose", action="store_true", dest="verbose", default=False, help="Verbose output."
|
|
)
|
|
parser.add_argument(
|
|
"-S", "--skip", type=int, dest="skip", default=0, help="Skip stories per month < #."
|
|
)
|
|
parser.add_argument(
|
|
"-w",
|
|
"--workerthreads",
|
|
type=int,
|
|
default=4,
|
|
help="Worker threads that will fetch feeds in parallel.",
|
|
)
|
|
|
|
def handle(self, *args, **options):
|
|
if options["daemonize"]:
|
|
daemonize()
|
|
|
|
settings.LOG_TO_STREAM = True
|
|
now = datetime.datetime.utcnow()
|
|
|
|
if options["skip"]:
|
|
feeds = Feed.objects.filter(
|
|
next_scheduled_update__lte=now, average_stories_per_month__lt=options["skip"], active=True
|
|
)
|
|
print(" ---> Skipping %s feeds" % feeds.count())
|
|
for feed in feeds:
|
|
feed.set_next_scheduled_update()
|
|
print(".", end=" ")
|
|
return
|
|
|
|
socket.setdefaulttimeout(options["timeout"])
|
|
if options["force"]:
|
|
feeds = Feed.objects.all()
|
|
elif options["username"]:
|
|
usersubs = UserSubscription.objects.filter(
|
|
user=User.objects.get(username=options["username"]), active=True
|
|
)
|
|
feeds = Feed.objects.filter(pk__in=usersubs.values("feed_id"))
|
|
elif options["feed"]:
|
|
feeds = Feed.objects.filter(pk=options["feed"])
|
|
else:
|
|
feeds = Feed.objects.filter(next_scheduled_update__lte=now, active=True)
|
|
|
|
feeds = feeds.order_by("?")
|
|
|
|
for f in feeds:
|
|
f.set_next_scheduled_update()
|
|
|
|
num_workers = min(len(feeds), options["workerthreads"])
|
|
if options["single_threaded"]:
|
|
num_workers = 1
|
|
|
|
options["compute_scores"] = True
|
|
options["quick"] = float(MStatistics.get("quick_fetch", 0))
|
|
options["updates_off"] = MStatistics.get("updates_off", False)
|
|
|
|
disp = feed_fetcher.Dispatcher(options, num_workers)
|
|
|
|
feeds_queue = []
|
|
for _ in range(num_workers):
|
|
feeds_queue.append([])
|
|
|
|
i = 0
|
|
for feed in feeds:
|
|
feeds_queue[i % num_workers].append(feed.pk)
|
|
i += 1
|
|
disp.add_jobs(feeds_queue, i)
|
|
|
|
django.db.connection.close()
|
|
|
|
print(" ---> Fetching %s feeds..." % feeds.count())
|
|
disp.run_jobs()
|