diff --git a/ansible/utils/check_droplet.py b/ansible/utils/check_droplet.py index 1abcc0d43..09747a9be 100644 --- a/ansible/utils/check_droplet.py +++ b/ansible/utils/check_droplet.py @@ -1,7 +1,8 @@ +import subprocess import sys import time + import digitalocean -import subprocess def test_ssh(drop): diff --git a/ansible/utils/generate_inventory.py b/ansible/utils/generate_inventory.py index 6fac02e79..4f74ffa92 100755 --- a/ansible/utils/generate_inventory.py +++ b/ansible/utils/generate_inventory.py @@ -1,9 +1,10 @@ #!/usr/bin/env python import os -import time -import sys import subprocess +import sys +import time + import digitalocean OLD = False diff --git a/api/newsblur.py b/api/newsblur.py index f417fe889..afa15341d 100644 --- a/api/newsblur.py +++ b/api/newsblur.py @@ -2,9 +2,10 @@ # Retooled by Samuel Clay, August 2011 # Modified by Luke Hagan, 2011-11-05 -import urllib.request, urllib.parse import http.cookiejar import json +import urllib.parse +import urllib.request __author__ = "Dananjaya Ramanayake , Samuel Clay " __version__ = "1.0" diff --git a/apps/analyzer/classifier.py b/apps/analyzer/classifier.py index 360472029..8e6c39c6f 100644 --- a/apps/analyzer/classifier.py +++ b/apps/analyzer/classifier.py @@ -1,7 +1,9 @@ -from apps.analyzer.models import Category, FeatureCategory -from django.db.models.aggregates import Sum import math +from django.db.models.aggregates import Sum + +from apps.analyzer.models import Category, FeatureCategory + class Classifier: def __init__(self, user, feed, phrases): diff --git a/apps/analyzer/feed_filter.py b/apps/analyzer/feed_filter.py index 7b3a6bbcf..ac5cba678 100644 --- a/apps/analyzer/feed_filter.py +++ b/apps/analyzer/feed_filter.py @@ -1,10 +1,12 @@ -from django.contrib.auth.models import User -from apps.rss_feeds.models import Feed -from apps.reader.models import UserSubscription -from apps.analyzer.models import Category, FeatureCategory import datetime -import re import math +import re + +from django.contrib.auth.models import User + +from apps.analyzer.models import Category, FeatureCategory +from apps.reader.models import UserSubscription +from apps.rss_feeds.models import Feed def entry_features(self, entry): diff --git a/apps/analyzer/forms.py b/apps/analyzer/forms.py index 6dc21e22e..de8d29dcc 100644 --- a/apps/analyzer/forms.py +++ b/apps/analyzer/forms.py @@ -1,12 +1,14 @@ import re + import requests from django import forms -from vendor.zebra.forms import StripePaymentForm -from django.utils.safestring import mark_safe from django.contrib.auth import authenticate from django.contrib.auth.models import User -from apps.profile.models import change_password, blank_authenticate, MGiftCode +from django.utils.safestring import mark_safe + +from apps.profile.models import MGiftCode, blank_authenticate, change_password from apps.social.models import MSocialProfile +from vendor.zebra.forms import StripePaymentForm class PopularityQueryForm(forms.Form): diff --git a/apps/analyzer/lda.py b/apps/analyzer/lda.py index 2d9d94f4b..5269f3c7e 100644 --- a/apps/analyzer/lda.py +++ b/apps/analyzer/lda.py @@ -1,12 +1,14 @@ -from bs4 import BeautifulSoup -from glob import glob -from collections import defaultdict -from math import log, exp -from random import random import zlib -from apps.rss_feeds.models import MStory +from collections import defaultdict +from glob import glob +from math import exp, log +from random import random + +from bs4 import BeautifulSoup from nltk import FreqDist +from apps.rss_feeds.models import MStory + def lgammln(xx): """ diff --git a/apps/analyzer/migrations/0001_initial.py b/apps/analyzer/migrations/0001_initial.py index a8b589883..19af099c6 100644 --- a/apps/analyzer/migrations/0001_initial.py +++ b/apps/analyzer/migrations/0001_initial.py @@ -1,8 +1,8 @@ # Generated by Django 2.0 on 2020-06-16 06:52 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/apps/analyzer/models.py b/apps/analyzer/models.py index b9eb99b6b..3bdd16e99 100644 --- a/apps/analyzer/models.py +++ b/apps/analyzer/models.py @@ -1,13 +1,15 @@ import datetime -import mongoengine as mongo from collections import defaultdict -from django.db import models -from django.contrib.auth.models import User -from django.template.loader import render_to_string -from django.core.mail import EmailMultiAlternatives + +import mongoengine as mongo from django.conf import settings -from apps.rss_feeds.models import Feed +from django.contrib.auth.models import User +from django.core.mail import EmailMultiAlternatives +from django.db import models +from django.template.loader import render_to_string + from apps.analyzer.tasks import EmailPopularityQuery +from apps.rss_feeds.models import Feed from utils import log as logging diff --git a/apps/analyzer/tests.py b/apps/analyzer/tests.py index ac1f3a1cf..8cbf1513a 100644 --- a/apps/analyzer/tests.py +++ b/apps/analyzer/tests.py @@ -1,14 +1,15 @@ -from django.test.client import Client -from apps.rss_feeds.models import MStory -from django.test import TestCase -from django.core import management +from itertools import groupby # from apps.analyzer.classifier import FisherClassifier import nltk -from itertools import groupby -from apps.analyzer.tokenizer import Tokenizer -from vendor.reverend.thomas import Bayes +from django.core import management +from django.test import TestCase +from django.test.client import Client + from apps.analyzer.phrase_filter import PhraseFilter +from apps.analyzer.tokenizer import Tokenizer +from apps.rss_feeds.models import MStory +from vendor.reverend.thomas import Bayes class QuadgramCollocationFinder(nltk.collocations.AbstractCollocationFinder): diff --git a/apps/analyzer/tfidf.py b/apps/analyzer/tfidf.py index 1bb015efe..9fdd98f60 100755 --- a/apps/analyzer/tfidf.py +++ b/apps/analyzer/tfidf.py @@ -6,8 +6,8 @@ Add your documents as two-element lists `[docname, [list_of_words_in_the_documen See the README for a usage example. """ -import sys import os +import sys class tfidf: diff --git a/apps/analyzer/urls.py b/apps/analyzer/urls.py index ed6580c67..21a91189f 100644 --- a/apps/analyzer/urls.py +++ b/apps/analyzer/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url + from apps.analyzer import views urlpatterns = [ diff --git a/apps/analyzer/views.py b/apps/analyzer/views.py index 8ea282565..66e738c69 100644 --- a/apps/analyzer/views.py +++ b/apps/analyzer/views.py @@ -1,19 +1,24 @@ import redis -from utils import log as logging -from django.shortcuts import get_object_or_404 -from django.views.decorators.http import require_POST from django.conf import settings -from django.shortcuts import render +from django.shortcuts import get_object_or_404, render +from django.views.decorators.http import require_POST from mongoengine.queryset import NotUniqueError -from apps.rss_feeds.models import Feed -from apps.reader.models import UserSubscription -from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag -from apps.analyzer.models import get_classifiers_for_user, MPopularityQuery + from apps.analyzer.forms import PopularityQueryForm +from apps.analyzer.models import ( + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, + MPopularityQuery, + get_classifiers_for_user, +) +from apps.reader.models import UserSubscription +from apps.rss_feeds.models import Feed from apps.social.models import MSocialSubscription from utils import json_functions as json -from utils.user_functions import get_user -from utils.user_functions import ajax_login_required +from utils import log as logging +from utils.user_functions import ajax_login_required, get_user def index(requst): diff --git a/apps/api/urls.py b/apps/api/urls.py index 3876b24e2..9a3603e16 100644 --- a/apps/api/urls.py +++ b/apps/api/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url + from apps.api import views urlpatterns = [ diff --git a/apps/categories/models.py b/apps/categories/models.py index 7254c80b3..02d3b1ad9 100644 --- a/apps/categories/models.py +++ b/apps/categories/models.py @@ -1,10 +1,12 @@ -import mongoengine as mongo from itertools import groupby -from apps.rss_feeds.models import Feed + +import mongoengine as mongo + from apps.reader.models import UserSubscription, UserSubscriptionFolders +from apps.rss_feeds.models import Feed from utils import json_functions as json -from utils.feed_functions import add_object_to_folder from utils import log as logging +from utils.feed_functions import add_object_to_folder class MCategory(mongo.Document): diff --git a/apps/categories/urls.py b/apps/categories/urls.py index 18ac85b27..77950af30 100644 --- a/apps/categories/urls.py +++ b/apps/categories/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url + from apps.categories import views urlpatterns = [ diff --git a/apps/feed_import/migrations/0001_initial.py b/apps/feed_import/migrations/0001_initial.py index 107f7a85d..c1e9d94c7 100644 --- a/apps/feed_import/migrations/0001_initial.py +++ b/apps/feed_import/migrations/0001_initial.py @@ -1,9 +1,10 @@ # Generated by Django 2.0 on 2020-06-16 06:52 import datetime + +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/apps/feed_import/models.py b/apps/feed_import/models.py index ad1e15fca..b34c7db11 100644 --- a/apps/feed_import/models.py +++ b/apps/feed_import/models.py @@ -1,20 +1,22 @@ -import datetime -import mongoengine as mongo -import pickle import base64 -from oauth2client.client import Error as OAuthError -from xml.etree.ElementTree import Element, SubElement, Comment, tostring -from lxml import etree -from django.db import models +import datetime +import pickle +from xml.etree.ElementTree import Comment, Element, SubElement, tostring + +import mongoengine as mongo from django.contrib.auth.models import User +from django.db import models +from lxml import etree from mongoengine.queryset import OperationError +from oauth2client.client import Error as OAuthError + import vendor.opml as opml -from apps.rss_feeds.models import Feed, DuplicateFeed from apps.reader.models import UserSubscription, UserSubscriptionFolders -from utils import json_functions as json, urlnorm +from apps.rss_feeds.models import DuplicateFeed, Feed +from utils import json_functions as json from utils import log as logging -from utils.feed_functions import timelimit -from utils.feed_functions import add_object_to_folder +from utils import urlnorm +from utils.feed_functions import add_object_to_folder, timelimit class OAuthToken(models.Model): diff --git a/apps/feed_import/tasks.py b/apps/feed_import/tasks.py index 7e70f5fd2..0742f9918 100644 --- a/apps/feed_import/tasks.py +++ b/apps/feed_import/tasks.py @@ -1,8 +1,9 @@ -from newsblur_web.celeryapp import app from django.contrib.auth.models import User -from apps.feed_import.models import UploadedOPML, OPMLImporter + +from apps.feed_import.models import OPMLImporter, UploadedOPML from apps.reader.models import UserSubscription from apps.social.models import MActivity +from newsblur_web.celeryapp import app from utils import log as logging diff --git a/apps/feed_import/test_feed_import.py b/apps/feed_import/test_feed_import.py index 57e429d9a..0f604182c 100644 --- a/apps/feed_import/test_feed_import.py +++ b/apps/feed_import/test_feed_import.py @@ -1,13 +1,15 @@ -import os -from django.test.client import Client -from django.test import TestCase -from django.contrib.auth.models import User -from django.urls import reverse -from apps.reader.models import UserSubscription, UserSubscriptionFolders -from apps.rss_feeds.models import merge_feeds, DuplicateFeed, Feed -from utils import json_functions as json_functions import json +import os + +from django.contrib.auth.models import User from django.core.management import call_command +from django.test import TestCase +from django.test.client import Client +from django.urls import reverse + +from apps.reader.models import UserSubscription, UserSubscriptionFolders +from apps.rss_feeds.models import DuplicateFeed, Feed, merge_feeds +from utils import json_functions as json_functions class Test_Import(TestCase): diff --git a/apps/feed_import/urls.py b/apps/feed_import/urls.py index feb2ace7f..3d1000016 100644 --- a/apps/feed_import/urls.py +++ b/apps/feed_import/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url + from apps.feed_import import views urlpatterns = [ diff --git a/apps/feed_import/views.py b/apps/feed_import/views.py index b6d28882b..cd6b8cb65 100644 --- a/apps/feed_import/views.py +++ b/apps/feed_import/views.py @@ -1,27 +1,28 @@ +import base64 import datetime import pickle -import base64 -from utils import log as logging -from oauth2client.client import OAuth2WebServerFlow, FlowExchangeError -from bson.errors import InvalidStringData import uuid -from django.contrib.sites.models import Site + +from bson.errors import InvalidStringData +from django.conf import settings +from django.contrib.auth import login as login_user from django.contrib.auth.models import User +from django.contrib.sites.models import Site # from django.db import IntegrityError from django.http import HttpResponse, HttpResponseRedirect -from django.conf import settings from django.urls import reverse -from django.contrib.auth import login as login_user from mongoengine.errors import ValidationError +from oauth2client.client import FlowExchangeError, OAuth2WebServerFlow + +from apps.feed_import.models import OAuthToken, OPMLExporter, OPMLImporter, UploadedOPML +from apps.feed_import.tasks import ProcessOPML from apps.reader.forms import SignupForm from apps.reader.models import UserSubscription -from apps.feed_import.models import OAuthToken -from apps.feed_import.models import OPMLImporter, OPMLExporter, UploadedOPML -from apps.feed_import.tasks import ProcessOPML from utils import json_functions as json -from utils.user_functions import ajax_login_required, get_user +from utils import log as logging from utils.feed_functions import TimeoutError +from utils.user_functions import ajax_login_required, get_user @ajax_login_required diff --git a/apps/mobile/urls.py b/apps/mobile/urls.py index 5d66dda53..38f857101 100644 --- a/apps/mobile/urls.py +++ b/apps/mobile/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url + from apps.mobile import views urlpatterns = [ diff --git a/apps/mobile/views.py b/apps/mobile/views.py index 95333ae27..de4ec45fe 100644 --- a/apps/mobile/views.py +++ b/apps/mobile/views.py @@ -1,8 +1,10 @@ -import os import base64 +import os + from django.conf import settings from django.http import HttpResponse from django.shortcuts import render + from apps.profile.models import Profile from apps.reader.models import UserSubscription, UserSubscriptionFolders from utils import json_functions as json diff --git a/apps/monitor/urls.py b/apps/monitor/urls.py index 4576c9b0a..e6f7bc673 100644 --- a/apps/monitor/urls.py +++ b/apps/monitor/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url + from apps.monitor.views import ( AppServers, AppTimes, @@ -7,6 +8,7 @@ from apps.monitor.views import ( Errors, FeedCounts, Feeds, + FeedSizes, LoadTimes, Stories, TasksCodes, @@ -15,7 +17,6 @@ from apps.monitor.views import ( TasksTimes, Updates, Users, - FeedSizes, ) urlpatterns = [ diff --git a/apps/monitor/views/newsblur_app_servers.py b/apps/monitor/views/newsblur_app_servers.py index 59f3e5984..11f9b76ae 100755 --- a/apps/monitor/views/newsblur_app_servers.py +++ b/apps/monitor/views/newsblur_app_servers.py @@ -1,7 +1,8 @@ import datetime + from django.conf import settings -from django.views import View from django.shortcuts import render +from django.views import View class AppServers(View): diff --git a/apps/monitor/views/newsblur_app_times.py b/apps/monitor/views/newsblur_app_times.py index 884058f0f..8970434c2 100755 --- a/apps/monitor/views/newsblur_app_times.py +++ b/apps/monitor/views/newsblur_app_times.py @@ -1,7 +1,8 @@ -from django.views import View -from django.shortcuts import render import datetime + from django.conf import settings +from django.shortcuts import render +from django.views import View class AppTimes(View): diff --git a/apps/monitor/views/newsblur_classifiers.py b/apps/monitor/views/newsblur_classifiers.py index 752fded2d..43bb5c376 100755 --- a/apps/monitor/views/newsblur_classifiers.py +++ b/apps/monitor/views/newsblur_classifiers.py @@ -1,6 +1,12 @@ -from django.views import View from django.shortcuts import render -from apps.analyzer.models import MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle +from django.views import View + +from apps.analyzer.models import ( + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, +) class Classifiers(View): diff --git a/apps/monitor/views/newsblur_feed_counts.py b/apps/monitor/views/newsblur_feed_counts.py index dbc355851..3a4d318cd 100755 --- a/apps/monitor/views/newsblur_feed_counts.py +++ b/apps/monitor/views/newsblur_feed_counts.py @@ -1,9 +1,10 @@ +import redis from django.conf import settings from django.shortcuts import render from django.views import View -import redis -from apps.rss_feeds.models import Feed, DuplicateFeed + from apps.push.models import PushSubscription +from apps.rss_feeds.models import DuplicateFeed, Feed from apps.statistics.models import MStatistics diff --git a/apps/monitor/views/newsblur_feed_sizes.py b/apps/monitor/views/newsblur_feed_sizes.py index c0da56eae..f4fade414 100644 --- a/apps/monitor/views/newsblur_feed_sizes.py +++ b/apps/monitor/views/newsblur_feed_sizes.py @@ -1,10 +1,11 @@ +import redis from django.conf import settings +from django.db.models import Sum from django.shortcuts import render from django.views import View -from django.db.models import Sum -import redis -from apps.rss_feeds.models import Feed, DuplicateFeed + from apps.push.models import PushSubscription +from apps.rss_feeds.models import DuplicateFeed, Feed from apps.statistics.models import MStatistics diff --git a/apps/monitor/views/newsblur_feeds.py b/apps/monitor/views/newsblur_feeds.py index fcacada32..a6432228c 100755 --- a/apps/monitor/views/newsblur_feeds.py +++ b/apps/monitor/views/newsblur_feeds.py @@ -1,8 +1,8 @@ -from django.views import View from django.shortcuts import render +from django.views import View -from apps.rss_feeds.models import Feed from apps.reader.models import UserSubscription +from apps.rss_feeds.models import Feed from apps.social.models import MSocialProfile, MSocialSubscription from apps.statistics.models import MStatistics diff --git a/apps/monitor/views/newsblur_stories.py b/apps/monitor/views/newsblur_stories.py index f88a5164c..ee73a196a 100755 --- a/apps/monitor/views/newsblur_stories.py +++ b/apps/monitor/views/newsblur_stories.py @@ -1,7 +1,7 @@ -from django.views import View from django.shortcuts import render -from apps.rss_feeds.models import MStory, MStarredStory -from apps.rss_feeds.models import MStory, MStarredStory +from django.views import View + +from apps.rss_feeds.models import MStarredStory, MStory class Stories(View): diff --git a/apps/monitor/views/newsblur_tasks_codes.py b/apps/monitor/views/newsblur_tasks_codes.py index 952a9afed..8bffd4c38 100755 --- a/apps/monitor/views/newsblur_tasks_codes.py +++ b/apps/monitor/views/newsblur_tasks_codes.py @@ -1,4 +1,5 @@ import datetime + from django.conf import settings from django.shortcuts import render from django.views import View diff --git a/apps/monitor/views/newsblur_updates.py b/apps/monitor/views/newsblur_updates.py index 2775e72ec..21fada112 100755 --- a/apps/monitor/views/newsblur_updates.py +++ b/apps/monitor/views/newsblur_updates.py @@ -1,5 +1,4 @@ import redis - from django.conf import settings from django.shortcuts import render from django.views import View diff --git a/apps/monitor/views/prometheus_redis.py b/apps/monitor/views/prometheus_redis.py index 9c2486ce8..50ffde3f6 100644 --- a/apps/monitor/views/prometheus_redis.py +++ b/apps/monitor/views/prometheus_redis.py @@ -1,8 +1,8 @@ import os import socket -from django.views import View from django.shortcuts import render +from django.views import View """ RedisActiveConnections diff --git a/apps/newsletters/models.py b/apps/newsletters/models.py index 31950ac2d..d37c6f901 100644 --- a/apps/newsletters/models.py +++ b/apps/newsletters/models.py @@ -1,21 +1,22 @@ import datetime import re + import redis +from django.conf import settings from django.contrib.sites.models import Site from django.core.mail import EmailMultiAlternatives -from django.urls import reverse -from django.conf import settings from django.template.loader import render_to_string +from django.urls import reverse from django.utils.html import linebreaks -from apps.rss_feeds.models import Feed, MStory, MFetchHistory -from apps.reader.models import UserSubscription, UserSubscriptionFolders -from apps.profile.models import Profile, MSentEmail -from apps.notifications.tasks import QueueNotifications -from apps.notifications.models import MUserFeedNotification +from apps.notifications.models import MUserFeedNotification +from apps.notifications.tasks import QueueNotifications +from apps.profile.models import MSentEmail, Profile +from apps.reader.models import UserSubscription, UserSubscriptionFolders +from apps.rss_feeds.models import Feed, MFetchHistory, MStory from utils import log as logging -from utils.story_functions import linkify from utils.scrubber import Scrubber +from utils.story_functions import linkify class EmailNewsletter: diff --git a/apps/newsletters/urls.py b/apps/newsletters/urls.py index e6263f189..39439a1ca 100644 --- a/apps/newsletters/urls.py +++ b/apps/newsletters/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url + from apps.newsletters import views urlpatterns = [ diff --git a/apps/newsletters/views.py b/apps/newsletters/views.py index f4ed35b1b..052c038ba 100644 --- a/apps/newsletters/views.py +++ b/apps/newsletters/views.py @@ -1,9 +1,11 @@ from pprint import pprint -from django.http import HttpResponse, Http404 + from django.conf import settings -from utils import log as logging +from django.http import Http404, HttpResponse + from apps.newsletters.models import EmailNewsletter from apps.rss_feeds.models import Feed, MStory +from utils import log as logging def newsletter_receive(request): diff --git a/apps/notifications/models.py b/apps/notifications/models.py index 3542415f4..9d158d69e 100644 --- a/apps/notifications/models.py +++ b/apps/notifications/models.py @@ -1,34 +1,36 @@ import datetime import enum import html -import redis import re +import urllib.parse + import mongoengine as mongo +import redis +from apns2.client import APNsClient +from apns2.errors import BadDeviceToken, DeviceTokenNotForTopic, Unregistered +from apns2.payload import Payload +from bs4 import BeautifulSoup from django.conf import settings from django.contrib.auth.models import User from django.contrib.sites.models import Site -from django.template.loader import render_to_string from django.core.mail import EmailMultiAlternatives +from django.template.loader import render_to_string -# from django.utils.html import strip_tags -from apps.rss_feeds.models import MStory, Feed -from apps.reader.models import UserSubscription from apps.analyzer.models import ( - MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag, + MClassifierTitle, + compute_story_score, ) -from apps.analyzer.models import compute_story_score -from utils.view_functions import is_true -from utils.story_functions import truncate_chars +from apps.reader.models import UserSubscription + +# from django.utils.html import strip_tags +from apps.rss_feeds.models import Feed, MStory from utils import log as logging from utils import mongoengine_fields -from apns2.errors import BadDeviceToken, Unregistered, DeviceTokenNotForTopic -from apns2.client import APNsClient -from apns2.payload import Payload -from bs4 import BeautifulSoup -import urllib.parse +from utils.story_functions import truncate_chars +from utils.view_functions import is_true class NotificationFrequency(enum.Enum): diff --git a/apps/notifications/tasks.py b/apps/notifications/tasks.py index cb67208a3..9a4fe4290 100644 --- a/apps/notifications/tasks.py +++ b/apps/notifications/tasks.py @@ -1,6 +1,7 @@ -from newsblur_web.celeryapp import app from django.contrib.auth.models import User + from apps.notifications.models import MUserFeedNotification +from newsblur_web.celeryapp import app from utils import log as logging diff --git a/apps/notifications/urls.py b/apps/notifications/urls.py index 53bd617fd..90a81f313 100644 --- a/apps/notifications/urls.py +++ b/apps/notifications/urls.py @@ -1,7 +1,8 @@ from django.conf.urls import url -from apps.notifications import views from oauth2_provider import views as op_views +from apps.notifications import views + urlpatterns = [ url(r"^$", views.notifications_by_feed, name="notifications-by-feed"), url(r"^feed/?$", views.set_notifications_for_feed, name="set-notifications-for-feed"), diff --git a/apps/notifications/views.py b/apps/notifications/views.py index 8c2662278..06f32101c 100644 --- a/apps/notifications/views.py +++ b/apps/notifications/views.py @@ -1,12 +1,13 @@ import redis from django.conf import settings from django.contrib.admin.views.decorators import staff_member_required -from utils import json_functions as json -from utils.user_functions import get_user, ajax_login_required + from apps.notifications.models import MUserFeedNotification, MUserNotificationTokens from apps.rss_feeds.models import Feed -from utils.view_functions import required_params +from utils import json_functions as json from utils import log as logging +from utils.user_functions import ajax_login_required, get_user +from utils.view_functions import required_params @ajax_login_required diff --git a/apps/oauth/urls.py b/apps/oauth/urls.py index 7dcb326e1..4875c790c 100644 --- a/apps/oauth/urls.py +++ b/apps/oauth/urls.py @@ -1,7 +1,8 @@ from django.conf.urls import url -from apps.oauth import views from oauth2_provider import views as op_views +from apps.oauth import views + urlpatterns = [ url(r"^twitter_connect/?$", views.twitter_connect, name="twitter-connect"), url(r"^facebook_connect/?$", views.facebook_connect, name="facebook-connect"), diff --git a/apps/oauth/views.py b/apps/oauth/views.py index c4374b21e..44eb305a9 100644 --- a/apps/oauth/views.py +++ b/apps/oauth/views.py @@ -1,27 +1,35 @@ -import urllib.request, urllib.parse, urllib.error import datetime +import urllib.error +import urllib.parse +import urllib.request + import lxml.html import tweepy +from django.conf import settings from django.contrib.auth.decorators import login_required -from django.urls import reverse from django.contrib.auth.models import User from django.contrib.sites.models import Site from django.http import HttpResponseForbidden, HttpResponseRedirect -from django.conf import settings -from mongoengine.queryset import NotUniqueError -from mongoengine.queryset import OperationError -from apps.social.models import MSocialServices, MSocialSubscription, MSharedStory -from apps.social.tasks import SyncTwitterFriends, SyncFacebookFriends -from apps.reader.models import UserSubscription, UserSubscriptionFolders, RUserStory -from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag -from apps.analyzer.models import compute_story_score -from apps.rss_feeds.models import Feed, MStory, MStarredStoryCounts, MStarredStory +from django.urls import reverse +from mongoengine.queryset import NotUniqueError, OperationError + +from apps.analyzer.models import ( + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, + compute_story_score, +) +from apps.reader.models import RUserStory, UserSubscription, UserSubscriptionFolders +from apps.rss_feeds.models import Feed, MStarredStory, MStarredStoryCounts, MStory from apps.rss_feeds.text_importer import TextImporter +from apps.social.models import MSharedStory, MSocialServices, MSocialSubscription +from apps.social.tasks import SyncFacebookFriends, SyncTwitterFriends +from utils import json_functions as json from utils import log as logging +from utils import urlnorm from utils.user_functions import ajax_login_required, oauth_login_required from utils.view_functions import render_to -from utils import urlnorm -from utils import json_functions as json from vendor import facebook diff --git a/apps/profile/factories.py b/apps/profile/factories.py index 3c20c2520..fba0246b8 100644 --- a/apps/profile/factories.py +++ b/apps/profile/factories.py @@ -1,6 +1,7 @@ import factory -from factory.django import DjangoModelFactory from django.contrib.auth.models import User +from factory.django import DjangoModelFactory + from apps.profile.models import Profile diff --git a/apps/profile/forms.py b/apps/profile/forms.py index f5adc12a3..6e4868026 100644 --- a/apps/profile/forms.py +++ b/apps/profile/forms.py @@ -1,12 +1,19 @@ import re + import requests from django import forms -from vendor.zebra.forms import StripePaymentForm -from django.utils.safestring import mark_safe from django.contrib.auth import authenticate from django.contrib.auth.models import User -from apps.profile.models import change_password, blank_authenticate, MGiftCode, MCustomStyling +from django.utils.safestring import mark_safe + +from apps.profile.models import ( + MCustomStyling, + MGiftCode, + blank_authenticate, + change_password, +) from apps.social.models import MSocialProfile +from vendor.zebra.forms import StripePaymentForm PLANS = [ ("newsblur-premium-36", mark_safe("$36 / year ($3/month)")), diff --git a/apps/profile/management/commands/check_db.py b/apps/profile/management/commands/check_db.py index c8aa983a2..7d2496a1d 100644 --- a/apps/profile/management/commands/check_db.py +++ b/apps/profile/management/commands/check_db.py @@ -1,4 +1,5 @@ import time + from django.core.management.base import BaseCommand from django.db import connections from django.db.utils import OperationalError diff --git a/apps/profile/management/commands/fp.py b/apps/profile/management/commands/fp.py index 1f7ed64f8..7dc22f990 100644 --- a/apps/profile/management/commands/fp.py +++ b/apps/profile/management/commands/fp.py @@ -1,5 +1,5 @@ -from django.core.management.base import BaseCommand from django.contrib.auth.models import User +from django.core.management.base import BaseCommand class Command(BaseCommand): diff --git a/apps/profile/management/commands/reimport_paypal_history.py b/apps/profile/management/commands/reimport_paypal_history.py index 4b6e943b4..eb853c4f7 100644 --- a/apps/profile/management/commands/reimport_paypal_history.py +++ b/apps/profile/management/commands/reimport_paypal_history.py @@ -1,11 +1,13 @@ -import stripe, datetime, time +import datetime +import time + +import stripe from django.conf import settings - -from django.core.management.base import BaseCommand from django.contrib.auth.models import User +from django.core.management.base import BaseCommand +from apps.profile.models import PaymentHistory, Profile from utils import log as logging -from apps.profile.models import Profile, PaymentHistory class Command(BaseCommand): diff --git a/apps/profile/management/commands/reimport_stripe_history.py b/apps/profile/management/commands/reimport_stripe_history.py index 6f8e803e4..5bb728c30 100644 --- a/apps/profile/management/commands/reimport_stripe_history.py +++ b/apps/profile/management/commands/reimport_stripe_history.py @@ -1,10 +1,12 @@ -import stripe, datetime, time -from django.conf import settings +import datetime +import time +import stripe +from django.conf import settings from django.core.management.base import BaseCommand -from utils import log as logging from apps.profile.models import Profile +from utils import log as logging class Command(BaseCommand): diff --git a/apps/profile/middleware.py b/apps/profile/middleware.py index 9710f466c..6fc33de1b 100644 --- a/apps/profile/middleware.py +++ b/apps/profile/middleware.py @@ -1,15 +1,17 @@ import datetime -import re import random +import re import time + import redis -from utils import log as logging -from django.http import HttpResponse from django.conf import settings from django.db import connection -from django.template import Template, Context +from django.http import HttpResponse +from django.template import Context, Template + from apps.statistics.rstats import round_time from utils import json_functions as json +from utils import log as logging class LastSeenMiddleware(object): diff --git a/apps/profile/migrations/0001_initial.py b/apps/profile/migrations/0001_initial.py index ac7841d50..6e50059d8 100644 --- a/apps/profile/migrations/0001_initial.py +++ b/apps/profile/migrations/0001_initial.py @@ -1,9 +1,11 @@ # Generated by Django 2.0 on 2020-06-16 06:52 import datetime + +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion + import vendor.timezones.fields diff --git a/apps/profile/migrations/0002_auto_20200620_0803.py b/apps/profile/migrations/0002_auto_20200620_0803.py index 945ec7d9f..7c5321a0e 100644 --- a/apps/profile/migrations/0002_auto_20200620_0803.py +++ b/apps/profile/migrations/0002_auto_20200620_0803.py @@ -1,8 +1,8 @@ # Generated by Django 2.0 on 2020-06-20 08:03 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/apps/profile/migrations/0003_auto_20201005_0932.py b/apps/profile/migrations/0003_auto_20201005_0932.py index db8f3c7a7..c5539d865 100644 --- a/apps/profile/migrations/0003_auto_20201005_0932.py +++ b/apps/profile/migrations/0003_auto_20201005_0932.py @@ -1,6 +1,7 @@ # Generated by Django 3.0.10 on 2020-10-05 09:32 from django.db import migrations + import vendor.timezones.fields diff --git a/apps/profile/migrations/0004_auto_20220110_2106.py b/apps/profile/migrations/0004_auto_20220110_2106.py index 676d762a5..5d53fb6f4 100644 --- a/apps/profile/migrations/0004_auto_20220110_2106.py +++ b/apps/profile/migrations/0004_auto_20220110_2106.py @@ -1,6 +1,7 @@ # Generated by Django 3.1.10 on 2022-01-10 21:06 from django.db import migrations, models + import vendor.timezones.fields diff --git a/apps/profile/migrations/0007_auto_20220125_2108.py b/apps/profile/migrations/0007_auto_20220125_2108.py index a456b15e3..c5472fc8b 100644 --- a/apps/profile/migrations/0007_auto_20220125_2108.py +++ b/apps/profile/migrations/0007_auto_20220125_2108.py @@ -1,6 +1,7 @@ # Generated by Django 3.1.10 on 2022-01-25 21:08 from django.db import migrations, models + import vendor.timezones.fields diff --git a/apps/profile/migrations/0009_paypalids.py b/apps/profile/migrations/0009_paypalids.py index 6480de3a3..a27cb1b3b 100644 --- a/apps/profile/migrations/0009_paypalids.py +++ b/apps/profile/migrations/0009_paypalids.py @@ -1,8 +1,8 @@ # Generated by Django 3.1.10 on 2022-02-08 23:15 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/apps/profile/migrations/0011_auto_20220408_1908.py b/apps/profile/migrations/0011_auto_20220408_1908.py index 187fbf0b0..91bcf9018 100644 --- a/apps/profile/migrations/0011_auto_20220408_1908.py +++ b/apps/profile/migrations/0011_auto_20220408_1908.py @@ -1,6 +1,7 @@ # Generated by Django 3.1.10 on 2022-04-08 19:08 from django.db import migrations, models + import vendor.timezones.fields diff --git a/apps/profile/migrations/0012_auto_20220511_1710.py b/apps/profile/migrations/0012_auto_20220511_1710.py index 582db6063..0257cafbc 100644 --- a/apps/profile/migrations/0012_auto_20220511_1710.py +++ b/apps/profile/migrations/0012_auto_20220511_1710.py @@ -1,6 +1,7 @@ # Generated by Django 3.1.10 on 2022-05-11 17:10 from django.db import migrations + import vendor.timezones.fields diff --git a/apps/profile/models.py b/apps/profile/models.py index 353619038..1226cce81 100644 --- a/apps/profile/models.py +++ b/apps/profile/models.py @@ -1,45 +1,45 @@ -import time import datetime -from wsgiref.util import application_uri -import dateutil -import stripe import hashlib import re -import redis +import time import uuid -import paypalrestsdk +from wsgiref.util import application_uri + +import dateutil import mongoengine as mongo -from django.db import models -from django.db import IntegrityError -from django.db.utils import DatabaseError -from django.db.models.signals import post_save -from django.db.models import Sum, Avg, Count -from django.db.models import Q +import paypalrestsdk +import redis +import stripe from django.conf import settings from django.contrib.auth import authenticate from django.contrib.auth.models import User from django.contrib.sites.models import Site -from django.core.mail import EmailMultiAlternatives -from django.core.mail import mail_admins -from django.urls import reverse +from django.core.mail import EmailMultiAlternatives, mail_admins +from django.db import IntegrityError, models +from django.db.models import Avg, Count, Q, Sum +from django.db.models.signals import post_save +from django.db.utils import DatabaseError from django.template.loader import render_to_string -from apps.rss_feeds.models import Feed, MStory, MStarredStory -from apps.rss_feeds.tasks import SchedulePremiumSetup -from apps.feed_import.models import OPMLExporter -from apps.reader.models import UserSubscription -from apps.reader.models import RUserStory -from utils import log as logging -from utils import json_functions as json -from utils.user_functions import generate_secret_token -from utils.feed_functions import chunks -from vendor.timezones.fields import TimeZoneField -from paypal.standard.ipn.signals import valid_ipn_received, invalid_ipn_received +from django.urls import reverse from paypal.standard.ipn.models import PayPalIPN -from zebra.signals import zebra_webhook_customer_subscription_created -from zebra.signals import zebra_webhook_customer_subscription_updated -from zebra.signals import zebra_webhook_charge_succeeded -from zebra.signals import zebra_webhook_charge_refunded -from zebra.signals import zebra_webhook_checkout_session_completed +from paypal.standard.ipn.signals import invalid_ipn_received, valid_ipn_received +from zebra.signals import ( + zebra_webhook_charge_refunded, + zebra_webhook_charge_succeeded, + zebra_webhook_checkout_session_completed, + zebra_webhook_customer_subscription_created, + zebra_webhook_customer_subscription_updated, +) + +from apps.feed_import.models import OPMLExporter +from apps.reader.models import RUserStory, UserSubscription +from apps.rss_feeds.models import Feed, MStarredStory, MStory +from apps.rss_feeds.tasks import SchedulePremiumSetup +from utils import json_functions as json +from utils import log as logging +from utils.feed_functions import chunks +from utils.user_functions import generate_secret_token +from vendor.timezones.fields import TimeZoneField class Profile(models.Model): @@ -167,8 +167,13 @@ class Profile(models.Model): except: logging.user(self.user, "~BR~SK~FWError cancelling premium renewal for: %s" % self.user.username) - from apps.social.models import MSocialProfile, MSharedStory, MSocialSubscription - from apps.social.models import MActivity, MInteraction + from apps.social.models import ( + MActivity, + MInteraction, + MSharedStory, + MSocialProfile, + MSocialSubscription, + ) try: social_profile = MSocialProfile.objects.get(user_id=self.user.pk) @@ -1420,7 +1425,7 @@ class Profile(models.Model): logging.user(self.user, "~BB~FM~SBSending OPML backup email to: %s" % self.user.email) def send_first_share_to_blurblog_email(self, force=False): - from apps.social.models import MSocialProfile, MSharedStory + from apps.social.models import MSharedStory, MSocialProfile if not self.user.email: return diff --git a/apps/profile/tasks.py b/apps/profile/tasks.py index 2179f0125..70cdb043a 100644 --- a/apps/profile/tasks.py +++ b/apps/profile/tasks.py @@ -1,9 +1,10 @@ import datetime -from newsblur_web.celeryapp import app + from apps.profile.models import Profile, RNewUserQueue -from utils import log as logging from apps.reader.models import UserSubscription, UserSubscriptionFolders -from apps.social.models import MSocialServices, MActivity, MInteraction +from apps.social.models import MActivity, MInteraction, MSocialServices +from newsblur_web.celeryapp import app +from utils import log as logging @app.task(name="email-new-user") diff --git a/apps/profile/test_profile.py b/apps/profile/test_profile.py index 15a1a9dd2..ac66ac37f 100644 --- a/apps/profile/test_profile.py +++ b/apps/profile/test_profile.py @@ -1,10 +1,11 @@ -from utils import json_functions as json -from django.test.client import Client -from django.test import TestCase -from django.urls import reverse from django.conf import settings +from django.test import TestCase +from django.test.client import Client +from django.urls import reverse from mongoengine.connection import connect, disconnect +from utils import json_functions as json + class Test_Profile(TestCase): fixtures = [ diff --git a/apps/profile/urls.py b/apps/profile/urls.py index 57f7cc519..ac5a0b949 100644 --- a/apps/profile/urls.py +++ b/apps/profile/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import * + from apps.profile import views urlpatterns = [ diff --git a/apps/profile/views.py b/apps/profile/views.py index 34faa2eb7..f07e7416c 100644 --- a/apps/profile/views.py +++ b/apps/profile/views.py @@ -1,41 +1,60 @@ -import re -import stripe -import requests import datetime +import json as python_json +import re + import dateutil -from django.contrib.auth.decorators import login_required -from django.views.decorators.http import require_POST -from django.views.decorators.csrf import csrf_protect, csrf_exempt -from django.contrib.auth import logout as logout_user +import requests +import stripe +from django.conf import settings +from django.contrib.admin.views.decorators import staff_member_required from django.contrib.auth import login as login_user +from django.contrib.auth import logout as logout_user +from django.contrib.auth.decorators import login_required +from django.contrib.auth.models import User +from django.contrib.sites.models import Site +from django.core.mail import mail_admins from django.db.models.aggregates import Sum from django.http import HttpResponse, HttpResponseRedirect -from django.contrib.sites.models import Site -from django.contrib.auth.models import User -from django.contrib.admin.views.decorators import staff_member_required -from django.urls import reverse from django.shortcuts import render -from django.core.mail import mail_admins -from django.conf import settings -from apps.profile.models import Profile, PaymentHistory, RNewUserQueue, MRedeemedCode, MGiftCode, PaypalIds -from apps.reader.models import UserSubscription, UserSubscriptionFolders, RUserStory -from apps.profile.forms import StripePlusPaymentForm, PLANS, DeleteAccountForm -from apps.profile.forms import ForgotPasswordForm, ForgotPasswordReturnForm, AccountSettingsForm -from apps.profile.forms import RedeemCodeForm -from apps.reader.forms import SignupForm, LoginForm -from apps.rss_feeds.models import MStarredStory, MStarredStoryCounts -from apps.social.models import MSocialServices, MActivity, MSocialProfile -from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag -from utils import json_functions as json -import json as python_json -from utils.user_functions import ajax_login_required -from utils.view_functions import render_to, is_true -from utils.user_functions import get_user -from utils import log as logging -from vendor.paypalapi.exceptions import PayPalAPIResponseError +from django.urls import reverse +from django.views.decorators.csrf import csrf_exempt, csrf_protect +from django.views.decorators.http import require_POST from paypal.standard.forms import PayPalPaymentsForm from paypal.standard.ipn.views import ipn as paypal_standard_ipn +from apps.analyzer.models import ( + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, +) +from apps.profile.forms import ( + PLANS, + AccountSettingsForm, + DeleteAccountForm, + ForgotPasswordForm, + ForgotPasswordReturnForm, + RedeemCodeForm, + StripePlusPaymentForm, +) +from apps.profile.models import ( + MGiftCode, + MRedeemedCode, + PaymentHistory, + PaypalIds, + Profile, + RNewUserQueue, +) +from apps.reader.forms import LoginForm, SignupForm +from apps.reader.models import RUserStory, UserSubscription, UserSubscriptionFolders +from apps.rss_feeds.models import MStarredStory, MStarredStoryCounts +from apps.social.models import MActivity, MSocialProfile, MSocialServices +from utils import json_functions as json +from utils import log as logging +from utils.user_functions import ajax_login_required, get_user +from utils.view_functions import is_true, render_to +from vendor.paypalapi.exceptions import PayPalAPIResponseError + INTEGER_FIELD_PREFS = ("feed_pane_size", "days_of_unread") SINGLE_FIELD_PREFS = ( "timezone", diff --git a/apps/push/migrations/0001_initial.py b/apps/push/migrations/0001_initial.py index 9baa20c60..3a7e2d495 100644 --- a/apps/push/migrations/0001_initial.py +++ b/apps/push/migrations/0001_initial.py @@ -1,8 +1,9 @@ # Generated by Django 2.0 on 2020-06-16 06:52 import datetime -from django.db import migrations, models + import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/apps/push/models.py b/apps/push/models.py index a128db5d1..8840a4c1c 100644 --- a/apps/push/models.py +++ b/apps/push/models.py @@ -1,19 +1,19 @@ # Adapted from djpubsubhubbub. See License: http://git.participatoryculture.org/djpubsubhubbub/tree/LICENSE +import hashlib +import re from datetime import datetime, timedelta + import feedparser import requests -import re - from django.conf import settings from django.db import models from django.urls import reverse -import hashlib from apps.push import signals from apps.rss_feeds.models import Feed from utils import log as logging -from utils.feed_functions import timelimit, TimeoutError +from utils.feed_functions import TimeoutError, timelimit DEFAULT_LEASE_SECONDS = 10 * 24 * 60 * 60 # 10 days diff --git a/apps/push/test_push.py b/apps/push/test_push.py index cfbef8bfe..c1cb03286 100644 --- a/apps/push/test_push.py +++ b/apps/push/test_push.py @@ -23,14 +23,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from datetime import datetime, timedelta import urllib +from datetime import datetime, timedelta -from django.urls import reverse from django.test import TestCase +from django.urls import reverse from apps.push.models import PushSubscription, PushSubscriptionManager -from apps.push.signals import pre_subscribe, verified, updated +from apps.push.signals import pre_subscribe, updated, verified class MockResponse(object): diff --git a/apps/push/urls.py b/apps/push/urls.py index 08a4d2a46..592014001 100644 --- a/apps/push/urls.py +++ b/apps/push/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import * + from apps.push import views urlpatterns = [ diff --git a/apps/push/views.py b/apps/push/views.py index 71451b80e..1d670ef66 100644 --- a/apps/push/views.py +++ b/apps/push/views.py @@ -1,10 +1,10 @@ # Adapted from djpubsubhubbub. See License: http://git.participatoryculture.org/djpubsubhubbub/tree/LICENSE -import feedparser -import random import datetime +import random -from django.http import HttpResponse, Http404 +import feedparser +from django.http import Http404, HttpResponse from django.http.request import UnreadablePostError from django.shortcuts import get_object_or_404 diff --git a/apps/reader/admin.py b/apps/reader/admin.py index fca4f8aca..2d1f2f2a5 100644 --- a/apps/reader/admin.py +++ b/apps/reader/admin.py @@ -1,6 +1,7 @@ -from apps.reader.models import UserSubscription, UserSubscriptionFolders, Feature from django.contrib import admin +from apps.reader.models import Feature, UserSubscription, UserSubscriptionFolders + admin.site.register(UserSubscription) admin.site.register(UserSubscriptionFolders) admin.site.register(Feature) diff --git a/apps/reader/factories.py b/apps/reader/factories.py index edba6c281..c4f8327ad 100644 --- a/apps/reader/factories.py +++ b/apps/reader/factories.py @@ -1,10 +1,11 @@ import factory -from factory.fuzzy import FuzzyAttribute from factory.django import DjangoModelFactory +from factory.fuzzy import FuzzyAttribute from faker import Faker -from apps.rss_feeds.factories import FeedFactory -from apps.reader.models import Feature, UserSubscription, UserSubscriptionFolders + from apps.profile.factories import UserFactory +from apps.reader.models import Feature, UserSubscription, UserSubscriptionFolders +from apps.rss_feeds.factories import FeedFactory fake = Faker() diff --git a/apps/reader/forms.py b/apps/reader/forms.py index a09cc0475..e7f563c9d 100644 --- a/apps/reader/forms.py +++ b/apps/reader/forms.py @@ -1,17 +1,24 @@ import datetime + from django import forms -from django.utils.translation import gettext_lazy as _ -from django.contrib.auth.models import User -from django.contrib.auth import authenticate -from django.db.models import Q from django.conf import settings -from apps.reader.models import Feature +from django.contrib.auth import authenticate +from django.contrib.auth.models import User +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ +from dns.resolver import ( + NXDOMAIN, + NoAnswer, + NoNameservers, + NoResolverConfiguration, + query, +) + +from apps.profile.models import RNewUserQueue, blank_authenticate from apps.profile.tasks import EmailNewUser +from apps.reader.models import Feature from apps.social.models import MActivity -from apps.profile.models import blank_authenticate, RNewUserQueue from utils import log as logging -from dns.resolver import query, NXDOMAIN, NoNameservers, NoAnswer -from dns.resolver import NoResolverConfiguration class LoginForm(forms.Form): diff --git a/apps/reader/managers.py b/apps/reader/managers.py index adafc88fa..a5a56083d 100644 --- a/apps/reader/managers.py +++ b/apps/reader/managers.py @@ -1,6 +1,8 @@ import sys -from django.db import models + from django.contrib.auth.models import User +from django.db import models + from apps.rss_feeds.models import DuplicateFeed from utils import log as logging diff --git a/apps/reader/migrations/0001_initial.py b/apps/reader/migrations/0001_initial.py index 61e86b9e5..386be088a 100644 --- a/apps/reader/migrations/0001_initial.py +++ b/apps/reader/migrations/0001_initial.py @@ -1,10 +1,12 @@ # Generated by Django 2.0 on 2020-06-16 06:52 -import apps.reader.models import datetime + +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion + +import apps.reader.models class Migration(migrations.Migration): diff --git a/apps/reader/models.py b/apps/reader/models.py index ca3d3b872..7a06bce4d 100644 --- a/apps/reader/models.py +++ b/apps/reader/models.py @@ -1,34 +1,37 @@ import datetime -import time import re -import redis -import pymongo -import celery -import mongoengine as mongo +import time from operator import itemgetter from pprint import pprint -from utils import log as logging -from utils import json_functions as json -from django.db import models, IntegrityError -from django.db.models import Q -from django.db.models import Count + +import celery +import mongoengine as mongo +import pymongo +import redis from django.conf import settings from django.contrib.auth.models import User from django.core.cache import cache +from django.db import IntegrityError, models +from django.db.models import Count, Q from django.template.defaultfilters import slugify -from mongoengine.queryset import OperationError -from mongoengine.queryset import NotUniqueError -from apps.reader.managers import UserSubscriptionManager -from apps.rss_feeds.models import Feed, MStory, DuplicateFeed -from apps.rss_feeds.tasks import NewFeeds -from apps.analyzer.models import MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle +from mongoengine.queryset import NotUniqueError, OperationError + from apps.analyzer.models import ( - apply_classifier_titles, - apply_classifier_feeds, + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, apply_classifier_authors, + apply_classifier_feeds, apply_classifier_tags, + apply_classifier_titles, ) from apps.analyzer.tfidf import tfidf +from apps.reader.managers import UserSubscriptionManager +from apps.rss_feeds.models import DuplicateFeed, Feed, MStory +from apps.rss_feeds.tasks import NewFeeds +from utils import json_functions as json +from utils import log as logging from utils.feed_functions import add_object_to_folder, chunks diff --git a/apps/reader/tasks.py b/apps/reader/tasks.py index 179500b6b..2c83929d7 100644 --- a/apps/reader/tasks.py +++ b/apps/reader/tasks.py @@ -1,10 +1,12 @@ import datetime -from newsblur_web.celeryapp import app -from utils import log as logging -from django.contrib.auth.models import User + from django.conf import settings +from django.contrib.auth.models import User + from apps.reader.models import UserSubscription from apps.social.models import MSocialSubscription +from newsblur_web.celeryapp import app +from utils import log as logging @app.task(name="freshen-homepage") diff --git a/apps/reader/test_reader.py b/apps/reader/test_reader.py index cd01cc84f..1eb5781f0 100644 --- a/apps/reader/test_reader.py +++ b/apps/reader/test_reader.py @@ -1,10 +1,11 @@ -from utils import json_functions as json -from django.test.client import Client -from django.test import TestCase -from django.urls import reverse from django.conf import settings +from django.test import TestCase +from django.test.client import Client +from django.urls import reverse from mongoengine.connection import connect, disconnect +from utils import json_functions as json + class Test_Reader(TestCase): fixtures = [ diff --git a/apps/reader/urls.py b/apps/reader/urls.py index f81ac9873..aac4fdbbc 100644 --- a/apps/reader/urls.py +++ b/apps/reader/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import * + from apps.reader import views urlpatterns = [ diff --git a/apps/recommendations/migrations/0001_initial.py b/apps/recommendations/migrations/0001_initial.py index 24217a40d..645d17caa 100644 --- a/apps/recommendations/migrations/0001_initial.py +++ b/apps/recommendations/migrations/0001_initial.py @@ -1,8 +1,8 @@ # Generated by Django 2.0 on 2020-06-16 06:52 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/apps/recommendations/models.py b/apps/recommendations/models.py index 43976f11b..afe00c3d1 100644 --- a/apps/recommendations/models.py +++ b/apps/recommendations/models.py @@ -1,11 +1,13 @@ -import mongoengine as mongo -from django.db import models -from django.contrib.auth.models import User -from apps.rss_feeds.models import Feed -from apps.reader.models import UserSubscription, UserSubscriptionFolders -from utils import json_functions as json from collections import defaultdict +import mongoengine as mongo +from django.contrib.auth.models import User +from django.db import models + +from apps.reader.models import UserSubscription, UserSubscriptionFolders +from apps.rss_feeds.models import Feed +from utils import json_functions as json + class RecommendedFeed(models.Model): feed = models.ForeignKey(Feed, related_name="recommendations", on_delete=models.CASCADE) diff --git a/apps/recommendations/templatetags/recommendations_tags.py b/apps/recommendations/templatetags/recommendations_tags.py index acd3d41fe..2df4d1bb4 100644 --- a/apps/recommendations/templatetags/recommendations_tags.py +++ b/apps/recommendations/templatetags/recommendations_tags.py @@ -1,9 +1,10 @@ import datetime -from django import template -from apps.reader.models import UserSubscription -from utils.user_functions import get_user -from apps.rss_feeds.models import MFeedIcon +from django import template + +from apps.reader.models import UserSubscription +from apps.rss_feeds.models import MFeedIcon +from utils.user_functions import get_user register = template.Library() diff --git a/apps/recommendations/urls.py b/apps/recommendations/urls.py index 9a61b3d5d..3899d33d8 100644 --- a/apps/recommendations/urls.py +++ b/apps/recommendations/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import * + from apps.recommendations import views urlpatterns = [ diff --git a/apps/recommendations/views.py b/apps/recommendations/views.py index dabdbd9c1..c5c1f99a1 100644 --- a/apps/recommendations/views.py +++ b/apps/recommendations/views.py @@ -1,13 +1,15 @@ -import re import datetime -from utils import log as logging +import re + from django.http import HttpResponse -from django.shortcuts import render, get_object_or_404 -from apps.recommendations.models import RecommendedFeed +from django.shortcuts import get_object_or_404, render + from apps.reader.models import UserSubscription +from apps.recommendations.models import RecommendedFeed from apps.rss_feeds.models import Feed, MFeedIcon from utils import json_functions as json -from utils.user_functions import get_user, ajax_login_required, admin_only +from utils import log as logging +from utils.user_functions import admin_only, ajax_login_required, get_user def load_recommended_feed(request): diff --git a/apps/rss_feeds/admin.py b/apps/rss_feeds/admin.py index c53b4799c..10a33a8a3 100644 --- a/apps/rss_feeds/admin.py +++ b/apps/rss_feeds/admin.py @@ -1,4 +1,5 @@ -from apps.rss_feeds.models import Feed from django.contrib import admin +from apps.rss_feeds.models import Feed + admin.site.register(Feed) diff --git a/apps/rss_feeds/factories.py b/apps/rss_feeds/factories.py index c6d6c0d62..613ca2a0d 100644 --- a/apps/rss_feeds/factories.py +++ b/apps/rss_feeds/factories.py @@ -1,9 +1,10 @@ -from faker import Faker import factory +from django.conf import settings from factory.django import DjangoModelFactory from factory.fuzzy import FuzzyAttribute +from faker import Faker + from apps.rss_feeds.models import DuplicateFeed, Feed -from django.conf import settings NEWSBLUR_DIR = settings.NEWSBLUR_DIR fake = Faker() diff --git a/apps/rss_feeds/management/commands/calculate_scores.py b/apps/rss_feeds/management/commands/calculate_scores.py index 7e371554a..f281404f6 100644 --- a/apps/rss_feeds/management/commands/calculate_scores.py +++ b/apps/rss_feeds/management/commands/calculate_scores.py @@ -1,11 +1,13 @@ -from django.core.management.base import BaseCommand -from apps.reader.models import UserSubscription +import datetime +import errno +import os +import re + from django.conf import settings from django.contrib.auth.models import User -import os -import errno -import re -import datetime +from django.core.management.base import BaseCommand + +from apps.reader.models import UserSubscription class Command(BaseCommand): diff --git a/apps/rss_feeds/management/commands/count_stories.py b/apps/rss_feeds/management/commands/count_stories.py index 41db438e6..b9679bc2e 100644 --- a/apps/rss_feeds/management/commands/count_stories.py +++ b/apps/rss_feeds/management/commands/count_stories.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand + from apps.rss_feeds.models import Feed diff --git a/apps/rss_feeds/management/commands/count_subscribers.py b/apps/rss_feeds/management/commands/count_subscribers.py index a9e465f7b..7a5adb0fd 100644 --- a/apps/rss_feeds/management/commands/count_subscribers.py +++ b/apps/rss_feeds/management/commands/count_subscribers.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand + from apps.rss_feeds.models import Feed diff --git a/apps/rss_feeds/management/commands/mark_read.py b/apps/rss_feeds/management/commands/mark_read.py index ae158ead4..29eda767a 100644 --- a/apps/rss_feeds/management/commands/mark_read.py +++ b/apps/rss_feeds/management/commands/mark_read.py @@ -1,8 +1,10 @@ -from django.core.management.base import BaseCommand -from django.contrib.auth.models import User -from apps.reader.models import UserSubscription import datetime +from django.contrib.auth.models import User +from django.core.management.base import BaseCommand + +from apps.reader.models import UserSubscription + class Command(BaseCommand): def add_arguments(self, parser): diff --git a/apps/rss_feeds/management/commands/query_popularity.py b/apps/rss_feeds/management/commands/query_popularity.py index 5afc87493..4e2b3ff59 100644 --- a/apps/rss_feeds/management/commands/query_popularity.py +++ b/apps/rss_feeds/management/commands/query_popularity.py @@ -1,12 +1,14 @@ -from django.core.management.base import BaseCommand -from apps.reader.models import UserSubscription +import datetime +import errno +import os +import re + from django.conf import settings from django.contrib.auth.models import User +from django.core.management.base import BaseCommand + +from apps.reader.models import UserSubscription from apps.rss_feeds.models import Feed -import os -import errno -import re -import datetime class Command(BaseCommand): diff --git a/apps/rss_feeds/management/commands/refresh_feed.py b/apps/rss_feeds/management/commands/refresh_feed.py index 14a1772c1..9dd4ab587 100644 --- a/apps/rss_feeds/management/commands/refresh_feed.py +++ b/apps/rss_feeds/management/commands/refresh_feed.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand + from apps.rss_feeds.models import Feed from utils.management_functions import daemonize diff --git a/apps/rss_feeds/management/commands/refresh_feeds.py b/apps/rss_feeds/management/commands/refresh_feeds.py index ef176beeb..3cac6d02d 100644 --- a/apps/rss_feeds/management/commands/refresh_feeds.py +++ b/apps/rss_feeds/management/commands/refresh_feeds.py @@ -1,15 +1,17 @@ -from django.core.management.base import BaseCommand +import datetime +import socket +from optparse import make_option + +import django from django.conf import settings from django.contrib.auth.models import User -from apps.statistics.models import MStatistics -from apps.rss_feeds.models import Feed +from django.core.management.base import BaseCommand + from apps.reader.models import UserSubscription -from optparse import make_option +from apps.rss_feeds.models import Feed +from apps.statistics.models import MStatistics from utils import feed_fetcher from utils.management_functions import daemonize -import django -import socket -import datetime class Command(BaseCommand): diff --git a/apps/rss_feeds/management/commands/task_feeds.py b/apps/rss_feeds/management/commands/task_feeds.py index 664c250ea..095ee896e 100644 --- a/apps/rss_feeds/management/commands/task_feeds.py +++ b/apps/rss_feeds/management/commands/task_feeds.py @@ -1,8 +1,10 @@ -from django.core.management.base import BaseCommand -from django.conf import settings -from apps.rss_feeds.tasks import TaskFeeds, TaskBrokenFeeds import datetime +from django.conf import settings +from django.core.management.base import BaseCommand + +from apps.rss_feeds.tasks import TaskBrokenFeeds, TaskFeeds + class Command(BaseCommand): def add_arguments(self, parser): diff --git a/apps/rss_feeds/management/commands/trim_feeds.py b/apps/rss_feeds/management/commands/trim_feeds.py index 2cff87ad8..d4e3522ea 100644 --- a/apps/rss_feeds/management/commands/trim_feeds.py +++ b/apps/rss_feeds/management/commands/trim_feeds.py @@ -1,7 +1,9 @@ -from django.core.management.base import BaseCommand -from apps.rss_feeds.models import Feed import gc +from django.core.management.base import BaseCommand + +from apps.rss_feeds.models import Feed + class Command(BaseCommand): def add_arguments(self, parser): diff --git a/apps/rss_feeds/migrations/0001_initial.py b/apps/rss_feeds/migrations/0001_initial.py index 5741ee8b5..c3beb2c42 100644 --- a/apps/rss_feeds/migrations/0001_initial.py +++ b/apps/rss_feeds/migrations/0001_initial.py @@ -1,7 +1,8 @@ # Generated by Django 2.0 on 2020-06-16 06:52 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models + import utils.fields diff --git a/apps/rss_feeds/migrations/0002_remove_mongo_types.py b/apps/rss_feeds/migrations/0002_remove_mongo_types.py index e34e70f9b..5acb061b9 100644 --- a/apps/rss_feeds/migrations/0002_remove_mongo_types.py +++ b/apps/rss_feeds/migrations/0002_remove_mongo_types.py @@ -1,7 +1,7 @@ # Generated by Django 3.1.4 on 2021-01-06 19:27 -from django.db import migrations from django.conf import settings +from django.db import migrations def remove_mongo_types(apps, schema_editor): diff --git a/apps/rss_feeds/migrations/0003_mongo_version_4_0.py b/apps/rss_feeds/migrations/0003_mongo_version_4_0.py index 7595452d3..816490137 100644 --- a/apps/rss_feeds/migrations/0003_mongo_version_4_0.py +++ b/apps/rss_feeds/migrations/0003_mongo_version_4_0.py @@ -1,7 +1,7 @@ # Generated by Django 3.1.10 on 2022-05-17 13:35 -from django.db import migrations from django.conf import settings +from django.db import migrations def set_mongo_feature_compatibility_version(apps, schema_editor): diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 89a0999a8..8fd347d5e 100755 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1,56 +1,63 @@ -import difflib -import bson -import requests +import base64 import datetime -import time +import difflib +import hashlib +import html +import math import random import re -import math -import mongoengine as mongo -import zlib -import hashlib -import redis -import base64 -import pymongo -import html +import time import urllib.parse +import zlib from collections import defaultdict from operator import itemgetter -from bson.objectid import ObjectId -from bs4 import BeautifulSoup -# from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder, TrigramAssocMeasures, BigramAssocMeasures -from django.db import models -from django.db import IntegrityError +import bson +import mongoengine as mongo +import pymongo +import redis +import requests +from bs4 import BeautifulSoup +from bson.objectid import ObjectId from django.conf import settings -from django.db.models.query import QuerySet -from django.db.utils import DatabaseError -from django.urls import reverse from django.contrib.auth.models import User from django.contrib.sites.models import Site + +# from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder, TrigramAssocMeasures, BigramAssocMeasures +from django.db import IntegrityError, models +from django.db.models.query import QuerySet +from django.db.utils import DatabaseError from django.template.defaultfilters import slugify -from django.utils.encoding import smart_bytes, smart_str -from django.utils.encoding import DjangoUnicodeDecodeError -from mongoengine.queryset import OperationError, Q, NotUniqueError +from django.urls import reverse +from django.utils.encoding import DjangoUnicodeDecodeError, smart_bytes, smart_str from mongoengine.errors import ValidationError -from vendor.timezones.utilities import localtime_for_timezone -from apps.rss_feeds.tasks import UpdateFeeds, PushFeeds, ScheduleCountTagsForUser +from mongoengine.queryset import NotUniqueError, OperationError, Q + +from apps.rss_feeds.tasks import PushFeeds, ScheduleCountTagsForUser, UpdateFeeds from apps.rss_feeds.text_importer import TextImporter -from apps.search.models import SearchStory, SearchFeed +from apps.search.models import SearchFeed, SearchStory from apps.statistics.rstats import RStats +from utils import feedfinder_forman, feedfinder_pilgrim from utils import json_functions as json -from utils import feedfinder_forman -from utils import feedfinder_pilgrim -from utils import urlnorm from utils import log as logging +from utils import urlnorm +from utils.feed_functions import ( + TimeoutError, + levenshtein_distance, + relative_timesince, + seconds_timesince, + timelimit, +) from utils.fields import AutoOneToOneField -from utils.feed_functions import levenshtein_distance -from utils.feed_functions import timelimit, TimeoutError -from utils.feed_functions import relative_timesince -from utils.feed_functions import seconds_timesince -from utils.story_functions import strip_tags, htmldiff, strip_comments, strip_comments__lxml -from utils.story_functions import prep_for_search -from utils.story_functions import create_imageproxy_signed_url +from utils.story_functions import ( + create_imageproxy_signed_url, + htmldiff, + prep_for_search, + strip_comments, + strip_comments__lxml, + strip_tags, +) +from vendor.timezones.utilities import localtime_for_timezone ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = list(range(4)) @@ -1223,7 +1230,12 @@ class Feed(models.Model): self.save(update_fields=["average_stories_per_month"]) def save_classifier_counts(self): - from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag + from apps.analyzer.models import ( + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, + ) def calculate_scores(cls, facet): map_f = """ diff --git a/apps/rss_feeds/test_rss_feeds.py b/apps/rss_feeds/test_rss_feeds.py index 6ad0dc764..4e167d816 100644 --- a/apps/rss_feeds/test_rss_feeds.py +++ b/apps/rss_feeds/test_rss_feeds.py @@ -1,13 +1,14 @@ import redis -from utils import json_functions as json -from django.test.client import Client -from django.test import TestCase -from django.core import management -from django.urls import reverse from django.conf import settings -from apps.rss_feeds.models import Feed, MStory +from django.core import management +from django.test import TestCase +from django.test.client import Client +from django.urls import reverse from mongoengine.connection import connect, disconnect +from apps.rss_feeds.models import Feed, MStory +from utils import json_functions as json + class Test_Feed(TestCase): fixtures = ["initial_data.json"] diff --git a/apps/rss_feeds/text_importer.py b/apps/rss_feeds/text_importer.py index 1f2771428..91436cd81 100644 --- a/apps/rss_feeds/text_importer.py +++ b/apps/rss_feeds/text_importer.py @@ -1,23 +1,24 @@ +import zlib +from socket import error as SocketError +from urllib.parse import urljoin + import requests import urllib3 -import zlib -from vendor import readability -from simplejson.decoder import JSONDecodeError -from requests.packages.urllib3.exceptions import LocationParseError -from socket import error as SocketError -from mongoengine.queryset import NotUniqueError +from bs4 import BeautifulSoup +from django.conf import settings +from django.contrib.sites.models import Site +from django.utils.encoding import smart_bytes, smart_str from lxml.etree import ParserError -from vendor.readability.readability import Unparseable -from utils import log as logging -from utils.feed_functions import timelimit, TimeoutError +from mongoengine.queryset import NotUniqueError from OpenSSL.SSL import Error as OpenSSLError from pyasn1.error import PyAsn1Error -from django.utils.encoding import smart_str -from django.conf import settings -from django.utils.encoding import smart_bytes -from django.contrib.sites.models import Site -from bs4 import BeautifulSoup -from urllib.parse import urljoin +from requests.packages.urllib3.exceptions import LocationParseError +from simplejson.decoder import JSONDecodeError + +from utils import log as logging +from utils.feed_functions import TimeoutError, timelimit +from vendor import readability +from vendor.readability.readability import Unparseable BROKEN_URLS = [ "gamespot.com", diff --git a/apps/rss_feeds/urls.py b/apps/rss_feeds/urls.py index 890b0c814..30d304f1f 100644 --- a/apps/rss_feeds/urls.py +++ b/apps/rss_feeds/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url + from apps.rss_feeds import views urlpatterns = [ diff --git a/apps/rss_feeds/views.py b/apps/rss_feeds/views.py index 5a062a6e5..0a98985a8 100644 --- a/apps/rss_feeds/views.py +++ b/apps/rss_feeds/views.py @@ -1,32 +1,34 @@ -import datetime import base64 -import redis +import datetime from urllib.parse import urlparse -from utils import log as logging -from django.shortcuts import get_object_or_404, render -from django.views.decorators.http import condition -from django.http import HttpResponseForbidden, HttpResponseRedirect, HttpResponse, Http404 + +import redis from django.conf import settings from django.contrib.auth.decorators import login_required from django.contrib.auth.models import User +from django.http import ( + Http404, + HttpResponse, + HttpResponseForbidden, + HttpResponseRedirect, +) +from django.shortcuts import get_object_or_404, render +from django.views.decorators.http import condition + +from apps.analyzer.models import get_classifiers_for_user +from apps.push.models import PushSubscription +from apps.reader.models import UserSubscription # from django.db import IntegrityError -from apps.rss_feeds.models import Feed, merge_feeds -from apps.rss_feeds.models import MFetchHistory -from apps.rss_feeds.models import MFeedIcon -from apps.push.models import PushSubscription -from apps.analyzer.models import get_classifiers_for_user -from apps.reader.models import UserSubscription -from apps.rss_feeds.models import MStory -from utils.user_functions import ajax_login_required -from utils import json_functions as json, feedfinder_forman as feedfinder -from utils.feed_functions import relative_timeuntil, relative_timesince -from utils.user_functions import get_user -from utils.view_functions import get_argument_or_404 -from utils.view_functions import required_params -from utils.view_functions import is_true -from vendor.timezones.utilities import localtime_for_timezone +from apps.rss_feeds.models import Feed, MFeedIcon, MFetchHistory, MStory, merge_feeds +from utils import feedfinder_forman as feedfinder +from utils import json_functions as json +from utils import log as logging +from utils.feed_functions import relative_timesince, relative_timeuntil from utils.ratelimit import ratelimit +from utils.user_functions import ajax_login_required, get_user +from utils.view_functions import get_argument_or_404, is_true, required_params +from vendor.timezones.utilities import localtime_for_timezone IGNORE_AUTOCOMPLETE = [ "facebook.com/feeds/notifications.php", diff --git a/apps/search/management/commands/index_feeds.py b/apps/search/management/commands/index_feeds.py index 92623d723..028d62aba 100644 --- a/apps/search/management/commands/index_feeds.py +++ b/apps/search/management/commands/index_feeds.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand + from apps.rss_feeds.models import Feed diff --git a/apps/search/management/commands/index_stories.py b/apps/search/management/commands/index_stories.py index 7a673b0c9..61b24c117 100644 --- a/apps/search/management/commands/index_stories.py +++ b/apps/search/management/commands/index_stories.py @@ -1,8 +1,10 @@ import re -from django.core.management.base import BaseCommand + from django.contrib.auth.models import User -from apps.rss_feeds.models import Feed, MStory +from django.core.management.base import BaseCommand + from apps.reader.models import UserSubscription +from apps.rss_feeds.models import Feed, MStory class Command(BaseCommand): diff --git a/apps/search/models.py b/apps/search/models.py index 59479f874..949c67542 100644 --- a/apps/search/models.py +++ b/apps/search/models.py @@ -1,19 +1,23 @@ +import datetime +import html import re import time -import datetime -import pymongo + +import celery import elasticsearch +import mongoengine as mongo +import pymongo import redis import urllib3 -import celery -import html -import mongoengine as mongo from django.conf import settings from django.contrib.auth.models import User -from apps.search.tasks import IndexSubscriptionsForSearch -from apps.search.tasks import FinishIndexSubscriptionsForSearch -from apps.search.tasks import IndexSubscriptionsChunkForSearch -from apps.search.tasks import IndexFeedsForSearch + +from apps.search.tasks import ( + FinishIndexSubscriptionsForSearch, + IndexFeedsForSearch, + IndexSubscriptionsChunkForSearch, + IndexSubscriptionsForSearch, +) from utils import log as logging from utils.feed_functions import chunks @@ -57,8 +61,8 @@ class MUserSearch(mongo.Document): # Should be run as a background task def index_subscriptions_for_search(self): - from apps.rss_feeds.models import Feed from apps.reader.models import UserSubscription + from apps.rss_feeds.models import Feed SearchStory.create_elasticsearch_mapping() @@ -169,8 +173,8 @@ class MUserSearch(mongo.Document): print(" ****> Error on search removal: %s" % e) def remove(self): - from apps.rss_feeds.models import Feed from apps.reader.models import UserSubscription + from apps.rss_feeds.models import Feed user = User.objects.get(pk=self.user_id) subscriptions = UserSubscription.objects.filter(user=self.user_id) @@ -680,6 +684,7 @@ class SearchFeed: @classmethod def export_csv(cls): import djqscsv + from apps.rss_feeds.models import Feed qs = Feed.objects.filter(num_subscribers__gte=20).values( diff --git a/apps/search/urls.py b/apps/search/urls.py index fcc841b86..284e6ba2c 100644 --- a/apps/search/urls.py +++ b/apps/search/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import * + from apps.search import views urlpatterns = [ diff --git a/apps/search/views.py b/apps/search/views.py index 37630b5ee..e723a0d04 100644 --- a/apps/search/views.py +++ b/apps/search/views.py @@ -1,9 +1,9 @@ -from apps.rss_feeds.models import Feed, MStory from apps.reader.models import UserSubscription +from apps.rss_feeds.models import Feed, MStory from apps.search.models import SearchStory from utils import json_functions as json +from utils.user_functions import ajax_login_required, get_user from utils.view_functions import required_params -from utils.user_functions import get_user, ajax_login_required # @required_params('story_hash') diff --git a/apps/social/management/commands/popular_stories.py b/apps/social/management/commands/popular_stories.py index 093dc049a..5747cddd8 100644 --- a/apps/social/management/commands/popular_stories.py +++ b/apps/social/management/commands/popular_stories.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand + from apps.social.models import MSharedStory diff --git a/apps/social/migrations/0001_username_unique.py b/apps/social/migrations/0001_username_unique.py index ba55f6399..c165ae0a1 100644 --- a/apps/social/migrations/0001_username_unique.py +++ b/apps/social/migrations/0001_username_unique.py @@ -1,8 +1,8 @@ # Generated by Django 3.1.4 on 2021-04-12 20:59 -from django.db import migrations -from django.conf import settings import pymongo +from django.conf import settings +from django.db import migrations def remove_unique_index(apps, schema_editor): diff --git a/apps/social/models.py b/apps/social/models.py index f4950a6a2..c34832e2c 100644 --- a/apps/social/models.py +++ b/apps/social/models.py @@ -1,49 +1,54 @@ -import os -import urllib.parse import datetime -import time -import zlib import hashlib import html as pyhtml -import redis -import re -import mongoengine as mongo -import random -import requests import html.parser as html_parser -import tweepy -import pynliner +import os +import random +import re +import time +import urllib.parse +import zlib from collections import defaultdict + +import mongoengine as mongo +import pynliner +import redis +import requests +import tweepy from bs4 import BeautifulSoup -from mongoengine.queryset import Q from django.conf import settings from django.contrib.auth.models import User from django.contrib.sites.models import Site -from django.urls import reverse -from django.template.loader import render_to_string -from django.template.defaultfilters import slugify from django.core.mail import EmailMultiAlternatives +from django.template.defaultfilters import slugify +from django.template.loader import render_to_string +from django.urls import reverse from django.utils.encoding import smart_bytes, smart_str -from apps.reader.models import UserSubscription, RUserStory -from apps.analyzer.models import MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle +from mongoengine.queryset import Q + from apps.analyzer.models import ( - apply_classifier_titles, - apply_classifier_feeds, + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, apply_classifier_authors, + apply_classifier_feeds, apply_classifier_tags, + apply_classifier_titles, ) +from apps.profile.models import MSentEmail, Profile +from apps.reader.models import RUserStory, UserSubscription from apps.rss_feeds.models import Feed, MStory -from apps.rss_feeds.text_importer import TextImporter from apps.rss_feeds.page_importer import PageImporter -from apps.profile.models import Profile, MSentEmail -from vendor import facebook -from utils import log as logging +from apps.rss_feeds.text_importer import TextImporter from utils import json_functions as json -from utils.feed_functions import relative_timesince, chunks -from utils.story_functions import truncate_chars, strip_tags, linkify +from utils import log as logging +from utils import s3_utils +from utils.feed_functions import chunks, relative_timesince from utils.image_functions import ImageOps from utils.scrubber import SelectiveScriptScrubber -from utils import s3_utils +from utils.story_functions import linkify, strip_tags, truncate_chars +from vendor import facebook try: from apps.social.spam import detect_spammers diff --git a/apps/social/tasks.py b/apps/social/tasks.py index 476450ad5..41eba160f 100644 --- a/apps/social/tasks.py +++ b/apps/social/tasks.py @@ -1,7 +1,13 @@ from bson.objectid import ObjectId -from newsblur_web.celeryapp import app -from apps.social.models import MSharedStory, MSocialProfile, MSocialServices, MSocialSubscription from django.contrib.auth.models import User + +from apps.social.models import ( + MSharedStory, + MSocialProfile, + MSocialServices, + MSocialSubscription, +) +from newsblur_web.celeryapp import app from utils import log as logging diff --git a/apps/social/templatetags/social_tags.py b/apps/social/templatetags/social_tags.py index 169995160..88a00b7d9 100644 --- a/apps/social/templatetags/social_tags.py +++ b/apps/social/templatetags/social_tags.py @@ -1,5 +1,6 @@ from django import template from django.conf import settings + from apps.social.models import MSocialProfile register = template.Library() diff --git a/apps/social/urls.py b/apps/social/urls.py index 1b5934d17..90b498b31 100644 --- a/apps/social/urls.py +++ b/apps/social/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url + from apps.social import views urlpatterns = [ diff --git a/apps/social/views.py b/apps/social/views.py index 513129020..669763fc2 100644 --- a/apps/social/views.py +++ b/apps/social/views.py @@ -1,48 +1,67 @@ -import time import datetime import random import re +import time + from bson.objectid import ObjectId -from mongoengine.queryset import NotUniqueError -from django.shortcuts import get_object_or_404, render -from django.urls import reverse +from django.conf import settings from django.contrib.auth.models import User from django.contrib.sites.models import Site +from django.http import ( + Http404, + HttpResponse, + HttpResponseForbidden, + HttpResponseRedirect, +) +from django.shortcuts import get_object_or_404, render from django.template.loader import render_to_string -from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseForbidden -from django.conf import settings +from django.urls import reverse from django.utils import feedgenerator -from apps.rss_feeds.models import MStory, Feed, MStarredStory -from apps.social.models import ( - MSharedStory, - MSocialServices, - MSocialProfile, - MSocialSubscription, - MCommentReply, -) -from apps.social.models import MInteraction, MActivity, MFollowRequest -from apps.social.tasks import PostToService, EmailCommentReplies, EmailStoryReshares -from apps.social.tasks import UpdateRecalcForSubscription, EmailFirstShare -from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag +from mongoengine.queryset import NotUniqueError + from apps.analyzer.models import ( - apply_classifier_titles, - apply_classifier_feeds, + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, apply_classifier_authors, + apply_classifier_feeds, apply_classifier_tags, + apply_classifier_titles, + get_classifiers_for_user, + sort_classifiers_by_feed, ) -from apps.analyzer.models import get_classifiers_for_user, sort_classifiers_by_feed -from apps.reader.models import UserSubscription from apps.profile.models import Profile +from apps.reader.models import UserSubscription +from apps.rss_feeds.models import Feed, MStarredStory, MStory +from apps.social.models import ( + MActivity, + MCommentReply, + MFollowRequest, + MInteraction, + MSharedStory, + MSocialProfile, + MSocialServices, + MSocialSubscription, +) +from apps.social.tasks import ( + EmailCommentReplies, + EmailFirstShare, + EmailStoryReshares, + PostToService, + UpdateRecalcForSubscription, +) +from utils import jennyholzer from utils import json_functions as json from utils import log as logging -from utils.user_functions import get_user, ajax_login_required -from utils.view_functions import render_to, is_true -from utils.view_functions import required_params -from utils.story_functions import format_story_link_date__short -from utils.story_functions import format_story_link_date__long -from utils.story_functions import strip_tags from utils.ratelimit import ratelimit -from utils import jennyholzer +from utils.story_functions import ( + format_story_link_date__long, + format_story_link_date__short, + strip_tags, +) +from utils.user_functions import ajax_login_required, get_user +from utils.view_functions import is_true, render_to, required_params from vendor.timezones.utilities import localtime_for_timezone diff --git a/apps/static/views.py b/apps/static/views.py index 934a38ef9..9b2f04df4 100644 --- a/apps/static/views.py +++ b/apps/static/views.py @@ -1,9 +1,11 @@ import os -import yaml + import redis +import yaml from django.conf import settings from django.http import HttpResponse from django.shortcuts import render + from apps.rss_feeds.models import Feed, MStory from apps.search.models import SearchFeed from utils import log as logging diff --git a/apps/statistics/management/commands/collect_feedback.py b/apps/statistics/management/commands/collect_feedback.py index 64ffa8d51..e8d9d8449 100644 --- a/apps/statistics/management/commands/collect_feedback.py +++ b/apps/statistics/management/commands/collect_feedback.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand + from apps.statistics.models import MFeedback diff --git a/apps/statistics/management/commands/collect_stats.py b/apps/statistics/management/commands/collect_stats.py index d664faf69..28e6e772b 100644 --- a/apps/statistics/management/commands/collect_stats.py +++ b/apps/statistics/management/commands/collect_stats.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand + from apps.statistics.models import MStatistics diff --git a/apps/statistics/models.py b/apps/statistics/models.py index 9d3df0934..05c7d4105 100644 --- a/apps/statistics/models.py +++ b/apps/statistics/models.py @@ -1,17 +1,21 @@ import datetime -import mongoengine as mongo -import urllib.request, urllib.error, urllib.parse -import redis +import urllib.error +import urllib.parse +import urllib.request + import dateutil +import mongoengine as mongo +import redis import requests from django.conf import settings -from apps.social.models import MSharedStory + from apps.profile.models import Profile +from apps.social.models import MSharedStory from apps.statistics.rstats import RStats, round_time -from utils.story_functions import relative_date -from utils import json_functions as json from utils import db_functions +from utils import json_functions as json from utils import log as logging +from utils.story_functions import relative_date class MStatistics(mongo.Document): diff --git a/apps/statistics/rstats.py b/apps/statistics/rstats.py index 3b8785d75..cb1b16568 100644 --- a/apps/statistics/rstats.py +++ b/apps/statistics/rstats.py @@ -1,7 +1,8 @@ -import redis import datetime import re from collections import defaultdict + +import redis from django.conf import settings diff --git a/apps/statistics/tasks.py b/apps/statistics/tasks.py index 79eb3ada9..2a27dd551 100644 --- a/apps/statistics/tasks.py +++ b/apps/statistics/tasks.py @@ -1,6 +1,5 @@ +from apps.statistics.models import MFeedback, MStatistics from newsblur_web.celeryapp import app -from apps.statistics.models import MStatistics -from apps.statistics.models import MFeedback from utils import log as logging diff --git a/apps/statistics/templatetags/statistics_tags.py b/apps/statistics/templatetags/statistics_tags.py index 9961ef9c1..362ec82a1 100644 --- a/apps/statistics/templatetags/statistics_tags.py +++ b/apps/statistics/templatetags/statistics_tags.py @@ -1,4 +1,5 @@ from django import template + from apps.statistics.models import MFeedback register = template.Library() diff --git a/apps/statistics/urls.py b/apps/statistics/urls.py index d0dc66881..8474204ac 100644 --- a/apps/statistics/urls.py +++ b/apps/statistics/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import * + from apps.statistics import views urlpatterns = [ diff --git a/apps/statistics/views.py b/apps/statistics/views.py index 0ce091b6b..0e79fb37d 100644 --- a/apps/statistics/views.py +++ b/apps/statistics/views.py @@ -1,20 +1,20 @@ import base64 -import pickle -import redis import datetime -from operator import countOf +import pickle from collections import defaultdict -from django.http import HttpResponse -from django.shortcuts import render -from django.contrib.auth.decorators import login_required -from django.contrib.auth.models import AnonymousUser -from django.contrib.auth.models import User +from operator import countOf + +import redis from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.contrib.auth.models import AnonymousUser, User +from django.http import HttpResponse, HttpResponseForbidden +from django.shortcuts import render from django.utils import feedgenerator -from django.http import HttpResponseForbidden -from apps.statistics.models import MStatistics, MFeedback -from apps.statistics.rstats import round_time + from apps.profile.models import PaymentHistory +from apps.statistics.models import MFeedback, MStatistics +from apps.statistics.rstats import round_time from utils import log as logging diff --git a/archive/ansible/do_inventory.py b/archive/ansible/do_inventory.py index 1766fb19f..aba12c107 100755 --- a/archive/ansible/do_inventory.py +++ b/archive/ansible/do_inventory.py @@ -131,10 +131,11 @@ import argparse import ast import os import re -import requests import sys from time import time +import requests + try: import ConfigParser except ImportError: diff --git a/archive/fabfile.py b/archive/fabfile.py index e8415cbfd..8df446eb2 100644 --- a/archive/fabfile.py +++ b/archive/fabfile.py @@ -1,22 +1,33 @@ -from fabric.api import cd, lcd, env, local, parallel, serial -from fabric.api import put, run, settings, sudo, prefix -from fabric.operations import prompt -from fabric.contrib import django -from fabric.contrib import files -from fabric.state import connections +import os +import re +import sys +import time +from collections import defaultdict +from contextlib import contextmanager as _contextmanager +from pprint import pprint + +import yaml +from boto.ec2.connection import EC2Connection # from fabric.colors import red, green, blue, cyan, magenta, white, yellow from boto.s3.connection import S3Connection from boto.s3.key import Key -from boto.ec2.connection import EC2Connection -import yaml -from pprint import pprint -from collections import defaultdict -from contextlib import contextmanager as _contextmanager -import os -import time -import sys -import re +from fabric.api import ( + cd, + env, + lcd, + local, + parallel, + prefix, + put, + run, + serial, + settings, + sudo, +) +from fabric.contrib import django, files +from fabric.operations import prompt +from fabric.state import connections # django.setup() diff --git a/archive/jammit.py b/archive/jammit.py index e3b83ecca..573f924ab 100644 --- a/archive/jammit.py +++ b/archive/jammit.py @@ -1,5 +1,6 @@ import os from fnmatch import fnmatch + import yaml from django.conf import settings diff --git a/archive/munin/aws_elb_latency b/archive/munin/aws_elb_latency index a1e8f6ee5..32b78683f 100755 --- a/archive/munin/aws_elb_latency +++ b/archive/munin/aws_elb_latency @@ -9,6 +9,7 @@ from boto.ec2.cloudwatch import CloudWatchConnection from vendor.munin import MuninPlugin + class AWSCloudWatchELBLatencyPlugin(MuninPlugin): category = "AWS" args = "-l 0 --base 1000" diff --git a/archive/munin/aws_elb_requests b/archive/munin/aws_elb_requests index 9af76a7a8..b0233864c 100755 --- a/archive/munin/aws_elb_requests +++ b/archive/munin/aws_elb_requests @@ -9,6 +9,7 @@ from boto.ec2.cloudwatch import CloudWatchConnection from vendor.munin import MuninPlugin + class AWSCloudWatchELBRequestsPlugin(MuninPlugin): category = "AWS" args = "-l 0 --base 1000" diff --git a/archive/munin/aws_sqs_queue_length_ b/archive/munin/aws_sqs_queue_length_ index 19219074d..35eee4890 100755 --- a/archive/munin/aws_sqs_queue_length_ +++ b/archive/munin/aws_sqs_queue_length_ @@ -8,6 +8,7 @@ from boto.sqs.connection import SQSConnection from vendor.munin import MuninPlugin + class AWSSQSQueueLengthPlugin(MuninPlugin): category = "AWS" args = "-l 0 --base 1000" diff --git a/archive/munin/cassandra_cfcounts b/archive/munin/cassandra_cfcounts index 4332ab16d..cf4fbdb53 100755 --- a/archive/munin/cassandra_cfcounts +++ b/archive/munin/cassandra_cfcounts @@ -1,8 +1,10 @@ #!/srv/newsblur/venv/newsblur3/bin/python import os + from vendor.munin.cassandra import MuninCassandraPlugin + class CassandraCFCountsPlugin(MuninCassandraPlugin): title = "read/write rate" args = "--base 1000 -l 0" diff --git a/archive/munin/cassandra_key_cache_ratio b/archive/munin/cassandra_key_cache_ratio index dbaa96449..c62da4c5a 100755 --- a/archive/munin/cassandra_key_cache_ratio +++ b/archive/munin/cassandra_key_cache_ratio @@ -1,8 +1,10 @@ #!/srv/newsblur/venv/newsblur3/bin/python import os + from vendor.munin.cassandra import MuninCassandraPlugin + class CassandraKeyCacheRatioPlugin(MuninCassandraPlugin): title = "key cache hit ratio" args = "--base 1000 -l 0" diff --git a/archive/munin/cassandra_latency b/archive/munin/cassandra_latency index 891a94294..af473ecac 100755 --- a/archive/munin/cassandra_latency +++ b/archive/munin/cassandra_latency @@ -1,8 +1,10 @@ #!/srv/newsblur/venv/newsblur3/bin/python import os + from vendor.munin.cassandra import MuninCassandraPlugin + class CassandraLatencyPlugin(MuninCassandraPlugin): title = "read/write latency" args = "--base 1000 -l 0" diff --git a/archive/munin/cassandra_load b/archive/munin/cassandra_load index 5a7916a49..4cfa5bb73 100755 --- a/archive/munin/cassandra_load +++ b/archive/munin/cassandra_load @@ -1,8 +1,10 @@ #!/srv/newsblur/venv/newsblur3/bin/python import os + from vendor.munin.cassandra import MuninCassandraPlugin + class CassandraLoadPlugin(MuninCassandraPlugin): title = "load (data stored in node)" args = "--base 1024 -l 0" diff --git a/archive/munin/cassandra_pending b/archive/munin/cassandra_pending index 92fddb700..0c7b71c9a 100755 --- a/archive/munin/cassandra_pending +++ b/archive/munin/cassandra_pending @@ -1,8 +1,10 @@ #!/srv/newsblur/venv/newsblur3/bin/python import os + from vendor.munin.cassandra import MuninCassandraPlugin + class CassandraPendingPlugin(MuninCassandraPlugin): title = "thread pool pending tasks" args = "--base 1000 -l 0" diff --git a/archive/munin/ddwrt_wl_rate b/archive/munin/ddwrt_wl_rate index 16e0e162c..5117359fb 100755 --- a/archive/munin/ddwrt_wl_rate +++ b/archive/munin/ddwrt_wl_rate @@ -2,6 +2,7 @@ from vendor.munin.ddwrt import DDWrtPlugin + class DDWrtWirelessRate(DDWrtPlugin): title = "Wireless rate" args = "--base 1000 -l 0" diff --git a/archive/munin/ddwrt_wl_signal b/archive/munin/ddwrt_wl_signal index 4f317f6ce..d0ae1fbec 100755 --- a/archive/munin/ddwrt_wl_signal +++ b/archive/munin/ddwrt_wl_signal @@ -2,6 +2,7 @@ from vendor.munin.ddwrt import DDWrtPlugin + class DDWrtWirelessSignalPlugin(DDWrtPlugin): title = "Wireless signal" args = "--base 1000 -l 0" diff --git a/archive/munin/gearman_connections b/archive/munin/gearman_connections index 207076c39..985d57b3b 100755 --- a/archive/munin/gearman_connections +++ b/archive/munin/gearman_connections @@ -2,6 +2,7 @@ from vendor.munin.gearman import MuninGearmanPlugin + class MuninGearmanConnectionsPlugin(MuninGearmanPlugin): title = "Gearman Connections" args = "--base 1000" diff --git a/archive/munin/gearman_queues b/archive/munin/gearman_queues index e0a2a308a..9941e99c1 100755 --- a/archive/munin/gearman_queues +++ b/archive/munin/gearman_queues @@ -1,8 +1,10 @@ #!/srv/newsblur/venv/newsblur3/bin/python import os + from vendor.munin.gearman import MuninGearmanPlugin + class MuninGearmanQueuesPlugin(MuninGearmanPlugin): title = "Gearman Queues" args = "--base 1000" diff --git a/archive/munin/hookbox b/archive/munin/hookbox index 4a136318e..2ceb00eff 100755 --- a/archive/munin/hookbox +++ b/archive/munin/hookbox @@ -1,14 +1,14 @@ #!/srv/newsblur/venv/newsblur3/bin/python -import os import json +import os import urllib + import urllib2 from vendor.munin import MuninPlugin - class HookboxPlugin(MuninPlugin): title = 'hookbox' args = "--base 1000" diff --git a/archive/munin/loadavg b/archive/munin/loadavg index f0b9da26f..d8629d85d 100755 --- a/archive/munin/loadavg +++ b/archive/munin/loadavg @@ -10,8 +10,10 @@ First tries reading from /proc/loadavg if it exists. Otherwise, execute """ import os + from vendor.munin import MuninPlugin + class LoadAVGPlugin(MuninPlugin): title = "Load average" args = "--base 1000 -l 0" @@ -35,7 +37,7 @@ class LoadAVGPlugin(MuninPlugin): if os.path.exists("/proc/loadavg"): loadavg = open("/proc/loadavg", "r").read().strip().split(' ') else: - from subprocess import Popen, PIPE + from subprocess import PIPE, Popen output = Popen(["uptime"], stdout=PIPE).communicate()[0] loadavg = output.rsplit(':', 1)[1].strip().split(' ')[:3] return dict(load=loadavg[1]) diff --git a/archive/munin/memcached_bytes b/archive/munin/memcached_bytes index 7b402253e..75b01261f 100755 --- a/archive/munin/memcached_bytes +++ b/archive/munin/memcached_bytes @@ -2,6 +2,7 @@ from vendor.munin.memcached import MuninMemcachedPlugin + class MuninMemcachedBytesPlugin(MuninMemcachedPlugin): title = "Memcached bytes read/written stats" args = "--base 1024" diff --git a/archive/munin/memcached_connections b/archive/munin/memcached_connections index 391a1299d..45adb6d5d 100755 --- a/archive/munin/memcached_connections +++ b/archive/munin/memcached_connections @@ -2,6 +2,7 @@ from vendor.munin.memcached import MuninMemcachedPlugin + class MuninMemcachedConnectionsPlugin(MuninMemcachedPlugin): title = "Memcached connections stats" args = "--base 1000" diff --git a/archive/munin/memcached_curr_items b/archive/munin/memcached_curr_items index 03b6e2759..9aeba345f 100755 --- a/archive/munin/memcached_curr_items +++ b/archive/munin/memcached_curr_items @@ -2,6 +2,7 @@ from vendor.munin.memcached import MuninMemcachedPlugin + class MuninMemcachedCurrentItemsPlugin(MuninMemcachedPlugin): title = "Memcached current items stats" args = "--base 1000" diff --git a/archive/munin/memcached_items b/archive/munin/memcached_items index 5ac10c7d6..f8fe4f715 100755 --- a/archive/munin/memcached_items +++ b/archive/munin/memcached_items @@ -2,6 +2,7 @@ from vendor.munin.memcached import MuninMemcachedPlugin + class MuninMemcachedItemsPlugin(MuninMemcachedPlugin): title = "Memcached new items stats" args = "--base 1000" diff --git a/archive/munin/memcached_queries b/archive/munin/memcached_queries index 0a40b0424..8deca8e6f 100755 --- a/archive/munin/memcached_queries +++ b/archive/munin/memcached_queries @@ -2,6 +2,7 @@ from vendor.munin.memcached import MuninMemcachedPlugin + class MuninMemcachedQueriesPlugin(MuninMemcachedPlugin): title = "Memcached query stats" args = "--base 1000" diff --git a/archive/munin/mongo_btree b/archive/munin/mongo_btree index 360c6f370..1c79b630e 100755 --- a/archive/munin/mongo_btree +++ b/archive/munin/mongo_btree @@ -2,9 +2,10 @@ ## GENERATED FILE - DO NOT EDIT -import urllib2 -import sys import os +import sys + +import urllib2 try: import json diff --git a/archive/munin/mongo_indexsize b/archive/munin/mongo_indexsize index 25baed35d..9294639bc 100644 --- a/archive/munin/mongo_indexsize +++ b/archive/munin/mongo_indexsize @@ -28,9 +28,10 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from pymongo import Connection import os +from pymongo import Connection + settings_host = os.environ.get("host", "127.0.0.1") settings_port = 27017 settings_db = 'newsblur' @@ -72,8 +73,8 @@ def doConfig(): if __name__ == "__main__": - from sys import argv from os import environ + from sys import argv # Could be done by a for loop # but i think if's are faster diff --git a/archive/munin/mongo_mem b/archive/munin/mongo_mem index 9853edc94..a7a31db13 100755 --- a/archive/munin/mongo_mem +++ b/archive/munin/mongo_mem @@ -2,9 +2,10 @@ ## GENERATED FILE - DO NOT EDIT -import urllib2 -import sys import os +import sys + +import urllib2 try: import json diff --git a/archive/munin/mongo_ops b/archive/munin/mongo_ops index 14fdebb7f..20bc91e58 100755 --- a/archive/munin/mongo_ops +++ b/archive/munin/mongo_ops @@ -2,9 +2,10 @@ ## GENERATED FILE - DO NOT EDIT -import urllib2 -import sys import os +import sys + +import urllib2 try: import json diff --git a/archive/munin/mongodb_conn b/archive/munin/mongodb_conn index 233436e3a..6e7d3672f 100755 --- a/archive/munin/mongodb_conn +++ b/archive/munin/mongodb_conn @@ -3,6 +3,7 @@ from vendor.munin.mongodb import MuninMongoDBPlugin + class MongoDBConnectionsPlugin(MuninMongoDBPlugin): args = "-l 0 --base 1000" vlabel = "count" diff --git a/archive/munin/mongodb_heap_usage b/archive/munin/mongodb_heap_usage index f19fb3d6a..8eed336a5 100755 --- a/archive/munin/mongodb_heap_usage +++ b/archive/munin/mongodb_heap_usage @@ -3,6 +3,7 @@ from vendor.munin.mongodb import MuninMongoDBPlugin + class MongoDBHeapUsagePlugin(MuninMongoDBPlugin): args = "-l 0 --base 1024" vlabel = "bytes" diff --git a/archive/munin/mongodb_objects_newsblur b/archive/munin/mongodb_objects_newsblur index dec54eb54..aeff0256f 100755 --- a/archive/munin/mongodb_objects_newsblur +++ b/archive/munin/mongodb_objects_newsblur @@ -3,6 +3,7 @@ from vendor.munin.mongodb import MuninMongoDBPlugin + class MongoDBObjectsPlugin(MuninMongoDBPlugin): args = "--base 1000" vlabel = "objects" diff --git a/archive/munin/mongodb_ops b/archive/munin/mongodb_ops index 203ed5aa7..e62219955 100755 --- a/archive/munin/mongodb_ops +++ b/archive/munin/mongodb_ops @@ -3,6 +3,7 @@ from vendor.munin.mongodb import MuninMongoDBPlugin + class MongoDBOpsPlugin(MuninMongoDBPlugin): args = "-l 0 --base 1000" vlabel = "ops / ${graph_period}" diff --git a/archive/munin/mongodb_page_faults b/archive/munin/mongodb_page_faults index cddb5b55d..f4c7d3eb8 100755 --- a/archive/munin/mongodb_page_faults +++ b/archive/munin/mongodb_page_faults @@ -3,6 +3,7 @@ from vendor.munin.mongodb import MuninMongoDBPlugin + class MongoDBPageFaultsPlugin(MuninMongoDBPlugin): args = "-l 0 --base 1000" vlabel = "page faults / sec" diff --git a/archive/munin/mongodb_queues b/archive/munin/mongodb_queues index 6c96b5773..69b87b3d5 100755 --- a/archive/munin/mongodb_queues +++ b/archive/munin/mongodb_queues @@ -3,6 +3,7 @@ from vendor.munin.mongodb import MuninMongoDBPlugin + class MongoDBQueuesPlugin(MuninMongoDBPlugin): args = "-l 0 --base 1000" vlabel = "count" diff --git a/archive/munin/mongodb_size_newsblur b/archive/munin/mongodb_size_newsblur index 09f9213e2..c38fbe764 100755 --- a/archive/munin/mongodb_size_newsblur +++ b/archive/munin/mongodb_size_newsblur @@ -3,6 +3,7 @@ from vendor.munin.mongodb import MuninMongoDBPlugin + class MongoDBSizePlugin(MuninMongoDBPlugin): args = "-l 0 --base 1024" vlabel = "bytes" diff --git a/archive/munin/munin/__init__.py b/archive/munin/munin/__init__.py index aa3462fc8..84b9e21c9 100755 --- a/archive/munin/munin/__init__.py +++ b/archive/munin/munin/__init__.py @@ -1,8 +1,8 @@ __version__ = "1.4" import os -import sys import socket +import sys from decimal import Decimal diff --git a/archive/munin/munin/cassandra.py b/archive/munin/munin/cassandra.py index 53ea4271f..9c9429b73 100755 --- a/archive/munin/munin/cassandra.py +++ b/archive/munin/munin/cassandra.py @@ -2,7 +2,7 @@ import os import re import socket import time -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen from vendor.munin import MuninPlugin diff --git a/archive/munin/munin/ddwrt.py b/archive/munin/munin/ddwrt.py index 2bebf3612..bfd5baf73 100755 --- a/archive/munin/munin/ddwrt.py +++ b/archive/munin/munin/ddwrt.py @@ -3,6 +3,7 @@ import os import re import urllib.request + from vendor.munin import MuninPlugin diff --git a/archive/munin/munin/gearman.py b/archive/munin/munin/gearman.py index 7165d3855..95854eb19 100755 --- a/archive/munin/munin/gearman.py +++ b/archive/munin/munin/gearman.py @@ -3,6 +3,7 @@ import os import re import socket + from vendor.munin import MuninPlugin worker_re = re.compile(r"^(?P\d+) (?P[\d\.]+) (?P[^\s]+) :\s?(?P.*)$") diff --git a/archive/munin/munin/memcached.py b/archive/munin/munin/memcached.py index 72d6590f1..e1de0f04f 100755 --- a/archive/munin/munin/memcached.py +++ b/archive/munin/munin/memcached.py @@ -2,6 +2,7 @@ import os import socket + from vendor.munin import MuninPlugin diff --git a/archive/munin/munin/mongodb.py b/archive/munin/munin/mongodb.py index 920805a15..adffb13ff 100755 --- a/archive/munin/munin/mongodb.py +++ b/archive/munin/munin/mongodb.py @@ -2,6 +2,7 @@ import os import sys + from vendor.munin import MuninPlugin diff --git a/archive/munin/munin/mysql.py b/archive/munin/munin/mysql.py index 542678cca..03d01ba3c 100755 --- a/archive/munin/munin/mysql.py +++ b/archive/munin/munin/mysql.py @@ -1,5 +1,8 @@ -import os, sys, re +import os +import re +import sys from configparser import SafeConfigParser + from vendor.munin import MuninPlugin diff --git a/archive/munin/munin/nginx.py b/archive/munin/munin/nginx.py index 3e386f8fc..b8918627a 100755 --- a/archive/munin/munin/nginx.py +++ b/archive/munin/munin/nginx.py @@ -3,6 +3,7 @@ import os import re import urllib.request + from vendor.munin import MuninPlugin diff --git a/archive/munin/munin/pgbouncer.py b/archive/munin/munin/pgbouncer.py index 4e9f14609..a41d27492 100755 --- a/archive/munin/munin/pgbouncer.py +++ b/archive/munin/munin/pgbouncer.py @@ -1,4 +1,5 @@ import sys + from vendor.munin.postgres import MuninPostgresPlugin diff --git a/archive/munin/munin/postgres.py b/archive/munin/munin/postgres.py index 42f7022ae..192079618 100755 --- a/archive/munin/munin/postgres.py +++ b/archive/munin/munin/postgres.py @@ -1,4 +1,6 @@ -import os, sys +import os +import sys + from vendor.munin import MuninPlugin diff --git a/archive/munin/munin/redis.py b/archive/munin/munin/redis.py index 00818075b..24b953c07 100755 --- a/archive/munin/munin/redis.py +++ b/archive/munin/munin/redis.py @@ -2,6 +2,7 @@ import os import socket + from vendor.munin import MuninPlugin diff --git a/archive/munin/munin/riak.py b/archive/munin/munin/riak.py index 9d15b27b0..3b3fa51f0 100755 --- a/archive/munin/munin/riak.py +++ b/archive/munin/munin/riak.py @@ -4,9 +4,11 @@ try: import json except ImportError: import simplejson as json + import os import sys import urllib.request + from vendor.munin import MuninPlugin diff --git a/archive/munin/mysql_dbrows_ b/archive/munin/mysql_dbrows_ index b5f2bdc5c..812620d89 100755 --- a/archive/munin/mysql_dbrows_ +++ b/archive/munin/mysql_dbrows_ @@ -3,6 +3,7 @@ from vendor.munin.mysql import MuninMySQLPlugin + class MuninMySQLDBRowsPlugin(MuninMySQLPlugin): dbname_in_args = True args = "-l 0 --base 1000" diff --git a/archive/munin/mysql_dbsize_ b/archive/munin/mysql_dbsize_ index 1c30b701b..a4b7dd9db 100755 --- a/archive/munin/mysql_dbsize_ +++ b/archive/munin/mysql_dbsize_ @@ -3,6 +3,7 @@ from vendor.munin.mysql import MuninMySQLPlugin + class MuninMySQLDBSizePlugin(MuninMySQLPlugin): dbname_in_args = True args = "-l 0 --base 1024" diff --git a/archive/munin/nginx_connections b/archive/munin/nginx_connections index ca7eabe3d..c4632cc5c 100755 --- a/archive/munin/nginx_connections +++ b/archive/munin/nginx_connections @@ -3,8 +3,10 @@ import os import re import urllib + from vendor.munin.nginx import MuninNginxPlugin + class MuninNginxConnectionsPlugin(MuninNginxPlugin): title = "Nginx Connections" args = "--base 1000" diff --git a/archive/munin/nginx_requests b/archive/munin/nginx_requests index 1ef06280d..ce846359c 100755 --- a/archive/munin/nginx_requests +++ b/archive/munin/nginx_requests @@ -3,8 +3,10 @@ import os import re import urllib + from vendor.munin.nginx import MuninNginxPlugin + class MuninNginxRequestsPlugin(MuninNginxPlugin): title = "Nginx Requests" args = "--base 1000" diff --git a/archive/munin/path_size b/archive/munin/path_size index bfa258e95..128f780d9 100755 --- a/archive/munin/path_size +++ b/archive/munin/path_size @@ -2,8 +2,10 @@ import os import subprocess + from vendor.munin import MuninPlugin + class PathSizePlugin(MuninPlugin): args = "--base 1024 -l 0" vlabel = "bytes" diff --git a/archive/munin/pgbouncer_pools_cl_ b/archive/munin/pgbouncer_pools_cl_ index 249929b58..12aa7ba01 100755 --- a/archive/munin/pgbouncer_pools_cl_ +++ b/archive/munin/pgbouncer_pools_cl_ @@ -3,6 +3,7 @@ from vendor.munin.pgbouncer import MuninPgBouncerPlugin + class MuninPgBouncerPoolsClientPlugin(MuninPgBouncerPlugin): command = "SHOW POOLS" vlabel = "Connections" diff --git a/archive/munin/pgbouncer_pools_sv_ b/archive/munin/pgbouncer_pools_sv_ index 09c6095c1..c794a11e6 100755 --- a/archive/munin/pgbouncer_pools_sv_ +++ b/archive/munin/pgbouncer_pools_sv_ @@ -3,6 +3,7 @@ from vendor.munin.pgbouncer import MuninPgBouncerPlugin + class MuninPgBouncerPoolsServerPlugin(MuninPgBouncerPlugin): command = "SHOW POOLS" vlabel = "Connections" diff --git a/archive/munin/pgbouncer_stats_avg_bytes_ b/archive/munin/pgbouncer_stats_avg_bytes_ index dbaaafd98..039105f91 100755 --- a/archive/munin/pgbouncer_stats_avg_bytes_ +++ b/archive/munin/pgbouncer_stats_avg_bytes_ @@ -3,6 +3,7 @@ from vendor.munin.pgbouncer import MuninPgBouncerPlugin + class MuninPgBouncerStatsBytesServerPlugin(MuninPgBouncerPlugin): command = "SHOW STATS" vlabel = "Bytes" diff --git a/archive/munin/pgbouncer_stats_avg_query_ b/archive/munin/pgbouncer_stats_avg_query_ index f913dc9c5..3124a4e21 100755 --- a/archive/munin/pgbouncer_stats_avg_query_ +++ b/archive/munin/pgbouncer_stats_avg_query_ @@ -3,6 +3,7 @@ from vendor.munin.pgbouncer import MuninPgBouncerPlugin + class MuninPgBouncerStatsQueryServerPlugin(MuninPgBouncerPlugin): command = "SHOW STATS" vlabel = "Microseconds" diff --git a/archive/munin/pgbouncer_stats_avg_req_ b/archive/munin/pgbouncer_stats_avg_req_ index 929e45589..90da415e5 100755 --- a/archive/munin/pgbouncer_stats_avg_req_ +++ b/archive/munin/pgbouncer_stats_avg_req_ @@ -3,6 +3,7 @@ from vendor.munin.pgbouncer import MuninPgBouncerPlugin + class MuninPgBouncerStatsRequestsServerPlugin(MuninPgBouncerPlugin): command = "SHOW STATS" vlabel = "Requests" diff --git a/archive/munin/postgres_block_read_ b/archive/munin/postgres_block_read_ index 55c673890..57d8d3e31 100755 --- a/archive/munin/postgres_block_read_ +++ b/archive/munin/postgres_block_read_ @@ -21,6 +21,7 @@ for a (short) description. from vendor.munin.postgres import MuninPostgresPlugin + class MuninPostgresBlockReadPlugin(MuninPostgresPlugin): dbname_in_args = True args = "--base 1000" diff --git a/archive/munin/postgres_commits_ b/archive/munin/postgres_commits_ index 39baf9b0b..c88b41139 100755 --- a/archive/munin/postgres_commits_ +++ b/archive/munin/postgres_commits_ @@ -24,6 +24,7 @@ Find out more at from vendor.munin.postgres import MuninPostgresPlugin + class MuninPostgresCommitsPlugin(MuninPostgresPlugin): dbname_in_args = True args = "--base 1000" diff --git a/archive/munin/postgres_connections b/archive/munin/postgres_connections index 8f5a49291..12641150a 100755 --- a/archive/munin/postgres_connections +++ b/archive/munin/postgres_connections @@ -2,6 +2,7 @@ from vendor.munin.postgres import MuninPostgresPlugin + class MuninPostgresConnectionsPlugin(MuninPostgresPlugin): dbname_in_args = False title = "Postgres active connections" diff --git a/archive/munin/postgres_locks b/archive/munin/postgres_locks index 80b8e42b4..45763a505 100755 --- a/archive/munin/postgres_locks +++ b/archive/munin/postgres_locks @@ -9,6 +9,7 @@ Show postgres lock statistics. from vendor.munin.postgres import MuninPostgresPlugin + class MuninPostgresLocksPlugin(MuninPostgresPlugin): dbname_in_args = False title = "Postgres locks" diff --git a/archive/munin/postgres_queries_ b/archive/munin/postgres_queries_ index 5cd2be6f8..fbf09519f 100755 --- a/archive/munin/postgres_queries_ +++ b/archive/munin/postgres_queries_ @@ -16,6 +16,7 @@ Find out more at from vendor.munin.postgres import MuninPostgresPlugin + class MuninPostgresQueriesPlugin(MuninPostgresPlugin): dbname_in_args = True args = "--base 1000" diff --git a/archive/munin/postgres_space_ b/archive/munin/postgres_space_ index d3ad6a001..4d1960564 100755 --- a/archive/munin/postgres_space_ +++ b/archive/munin/postgres_space_ @@ -9,6 +9,7 @@ Plugin to monitor PostgreSQL disk usage. from vendor.munin.postgres import MuninPostgresPlugin + class MuninPostgresSpacePlugin(MuninPostgresPlugin): dbname_in_args = True args = "-l 0 --base 1024" diff --git a/archive/munin/postgres_table_sizes b/archive/munin/postgres_table_sizes index c174d52c3..6ea465129 100755 --- a/archive/munin/postgres_table_sizes +++ b/archive/munin/postgres_table_sizes @@ -5,6 +5,7 @@ from vendor.munin.postgres import MuninPostgresPlugin + class PostgresTableSizes(MuninPostgresPlugin): vlabel = "Table Size" title = "Table Sizes" diff --git a/archive/munin/redis_active_connections b/archive/munin/redis_active_connections index ada659f48..5bd1d51a8 100755 --- a/archive/munin/redis_active_connections +++ b/archive/munin/redis_active_connections @@ -2,6 +2,7 @@ from vendor.munin.redis import MuninRedisPlugin + class MuninRedisActiveConnectionsPlugin(MuninRedisPlugin): title = "Redis active connections" args = "--base 1000" diff --git a/archive/munin/redis_commands b/archive/munin/redis_commands index 0e86805d9..7ba967d44 100755 --- a/archive/munin/redis_commands +++ b/archive/munin/redis_commands @@ -2,6 +2,7 @@ from vendor.munin.redis import MuninRedisPlugin + class MuninRedisCommandsPlugin(MuninRedisPlugin): title = "Redis commands" args = "--base 1000" diff --git a/archive/munin/redis_connects b/archive/munin/redis_connects index f56519a90..a51c9db77 100755 --- a/archive/munin/redis_connects +++ b/archive/munin/redis_connects @@ -2,6 +2,7 @@ from vendor.munin.redis import MuninRedisPlugin + class MuninRedisTotalConnectionsPlugin(MuninRedisPlugin): title = "Redis connects" args = "--base 1000" diff --git a/archive/munin/redis_size b/archive/munin/redis_size index dcdf309b9..dc87aeaf3 100755 --- a/archive/munin/redis_size +++ b/archive/munin/redis_size @@ -1,6 +1,10 @@ #!/srv/newsblur/venv/newsblur3/bin/python +import os + +import redis + from utils.munin.base import MuninGraph -import os, redis + class NBMuninGraph(MuninGraph): diff --git a/archive/munin/redis_used_memory b/archive/munin/redis_used_memory index 86bb32ba4..3b5d6cb63 100755 --- a/archive/munin/redis_used_memory +++ b/archive/munin/redis_used_memory @@ -2,6 +2,7 @@ from vendor.munin.redis import MuninRedisPlugin + class MuninRedisUsedMemoryPlugin(MuninRedisPlugin): title = "Redis used memory" args = "--base 1024" diff --git a/archive/munin/request_time b/archive/munin/request_time index f8c7a0714..edabb1baa 100755 --- a/archive/munin/request_time +++ b/archive/munin/request_time @@ -2,10 +2,12 @@ import os from time import time -from urllib2 import urlopen, Request + +from urllib2 import Request, urlopen from vendor.munin import MuninPlugin + class MuninRequestTimePlugin(MuninPlugin): title = "Request Time" args = "--base 1000" diff --git a/archive/munin/riak_ops b/archive/munin/riak_ops index 372d6270f..8470f37d6 100755 --- a/archive/munin/riak_ops +++ b/archive/munin/riak_ops @@ -3,6 +3,7 @@ from vendor.munin.riak import MuninRiakPlugin + class RiakOpsPlugin(MuninRiakPlugin): args = "-l 0 --base 1000" vlabel = "ops/sec" diff --git a/archive/munin/tc_size b/archive/munin/tc_size index bb3f49cef..e21b6a125 100755 --- a/archive/munin/tc_size +++ b/archive/munin/tc_size @@ -5,6 +5,7 @@ import subprocess from vendor.munin import MuninPlugin + class MuninTokyoCabinetSizePlugin(MuninPlugin): title = "Size of Tokyo Cabinet database" args = "--base 1024" diff --git a/flask_metrics/flask_metrics_haproxy.py b/flask_metrics/flask_metrics_haproxy.py index 86cb4ccd2..106594589 100644 --- a/flask_metrics/flask_metrics_haproxy.py +++ b/flask_metrics/flask_metrics_haproxy.py @@ -1,9 +1,10 @@ -from flask import Flask, render_template, Response -from newsblur_web import settings -import sentry_sdk -from sentry_sdk.integrations.flask import FlaskIntegration import requests +import sentry_sdk +from flask import Flask, Response, render_template from requests.auth import HTTPBasicAuth +from sentry_sdk.integrations.flask import FlaskIntegration + +from newsblur_web import settings if settings.FLASK_SENTRY_DSN is not None: sentry_sdk.init( diff --git a/newsblur_web/celeryapp.py b/newsblur_web/celeryapp.py index e49929029..0f22dac8d 100644 --- a/newsblur_web/celeryapp.py +++ b/newsblur_web/celeryapp.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, unicode_literals + import os + from celery import Celery from django.apps import apps diff --git a/newsblur_web/urls.py b/newsblur_web/urls.py index dd5950fe6..309ad76fe 100644 --- a/newsblur_web/urls.py +++ b/newsblur_web/urls.py @@ -1,13 +1,14 @@ -from django.conf.urls import include, url from django.conf import settings -from apps.reader import views as reader_views -from apps.social import views as social_views -from apps.static import views as static_views -from apps.profile import views as profile_views +from django.conf.urls import include, url from django.conf.urls.static import static from django.contrib import admin from django.contrib.auth.views import LogoutView +from apps.profile import views as profile_views +from apps.reader import views as reader_views +from apps.social import views as social_views +from apps.static import views as static_views + admin.autodiscover() urlpatterns = [ diff --git a/perf/locust.py b/perf/locust.py index 15d748547..b9bd6a699 100644 --- a/perf/locust.py +++ b/perf/locust.py @@ -1,7 +1,8 @@ -import time -from locust import HttpUser, task, between import os +import time + import requests +from locust import HttpUser, between, task class NB_PerfTest(HttpUser): diff --git a/utils/S3.py b/utils/S3.py index ca76850eb..489d4dd8a 100644 --- a/utils/S3.py +++ b/utils/S3.py @@ -13,13 +13,15 @@ import base64 import hmac import http.client import re -import sha import sys import time -import urllib.request, urllib.parse, urllib.error +import urllib.error import urllib.parse +import urllib.request import xml.sax +import sha + DEFAULT_HOST = "s3.amazonaws.com" PORTS_BY_SECURITY = {True: 443, False: 80} METADATA_PREFIX = "x-amz-meta-" diff --git a/utils/archive/Image Color Algorithm.py b/utils/archive/Image Color Algorithm.py index d6fc230f4..01c119227 100644 --- a/utils/archive/Image Color Algorithm.py +++ b/utils/archive/Image Color Algorithm.py @@ -1,7 +1,8 @@ -from PIL import Image +from pprint import pprint + import scipy import scipy.cluster -from pprint import pprint +from PIL import Image image = Image.open("logo.png") NUM_CLUSTERS = 5 diff --git a/utils/archive/bootstrap_intel.py b/utils/archive/bootstrap_intel.py index 357fe5401..e5b350c08 100644 --- a/utils/archive/bootstrap_intel.py +++ b/utils/archive/bootstrap_intel.py @@ -1,10 +1,14 @@ import sys -from mongoengine.queryset import OperationError + from mongoengine.errors import ValidationError -from apps.analyzer.models import MClassifierFeed -from apps.analyzer.models import MClassifierAuthor -from apps.analyzer.models import MClassifierTag -from apps.analyzer.models import MClassifierTitle +from mongoengine.queryset import OperationError + +from apps.analyzer.models import ( + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, +) for classifier_cls in [MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle]: print(" ================================================================= ") diff --git a/utils/archive/bootstrap_mongo.py b/utils/archive/bootstrap_mongo.py index 5df95f55d..4162ec0a1 100644 --- a/utils/archive/bootstrap_mongo.py +++ b/utils/archive/bootstrap_mongo.py @@ -1,12 +1,19 @@ -from pprint import pprint -from django.conf import settings -from apps.reader.models import MUserStory -from apps.rss_feeds.models import Feed, MStory, MFeedPage -from apps.rss_feeds.models import MFeedIcon, FeedIcon -from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag -import mongoengine, pymongo import sys +from pprint import pprint + +import mongoengine +import pymongo +from django.conf import settings from mongoengine.queryset import OperationError + +from apps.analyzer.models import ( + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, +) +from apps.reader.models import MUserStory +from apps.rss_feeds.models import Feed, FeedIcon, MFeedIcon, MFeedPage, MStory from utils import json_functions as json MONGO_DB = settings.MONGO_DB diff --git a/utils/archive/bootstrap_redis_sessions.py b/utils/archive/bootstrap_redis_sessions.py index 1359415a6..f718cf084 100644 --- a/utils/archive/bootstrap_redis_sessions.py +++ b/utils/archive/bootstrap_redis_sessions.py @@ -1,4 +1,5 @@ import math + import redis from django.conf import settings from django.contrib.sessions.models import Session diff --git a/utils/archive/bootstrap_story_hash.py b/utils/archive/bootstrap_story_hash.py index 7d856670d..266ccb30f 100644 --- a/utils/archive/bootstrap_story_hash.py +++ b/utils/archive/bootstrap_story_hash.py @@ -1,7 +1,9 @@ import time + import pymongo from django.conf import settings -from apps.rss_feeds.models import MStory, Feed + +from apps.rss_feeds.models import Feed, MStory db = settings.MONGODB batch = 0 diff --git a/utils/archive/check_status.py b/utils/archive/check_status.py index b5e20d707..5dd6373d3 100644 --- a/utils/archive/check_status.py +++ b/utils/archive/check_status.py @@ -1,4 +1,5 @@ import time + import requests url = "http://www.newsblur.com" diff --git a/utils/archive/green.py b/utils/archive/green.py index 7f746cd85..9c83b734a 100644 --- a/utils/archive/green.py +++ b/utils/archive/green.py @@ -2,10 +2,14 @@ from gevent import monkey monkey.patch_socket() -from newsblur.utils import feedparser +import urllib.error +import urllib.parse +import urllib.request + import gevent from gevent import queue -import urllib.request, urllib.error, urllib.parse + +from newsblur.utils import feedparser def fetch_title(url): diff --git a/utils/archive/memcached_status.py b/utils/archive/memcached_status.py index 62f426371..3d9dfd17c 100644 --- a/utils/archive/memcached_status.py +++ b/utils/archive/memcached_status.py @@ -1,6 +1,7 @@ -import memcache import re import sys + +import memcache from settings import CACHE_BACKEND # gfranxman diff --git a/utils/backups/backup_mongo.py b/utils/backups/backup_mongo.py index f795d516a..236046b39 100755 --- a/utils/backups/backup_mongo.py +++ b/utils/backups/backup_mongo.py @@ -1,14 +1,16 @@ #!/usr/bin/python3 -from datetime import datetime, timedelta -import os -import sys -import re import logging import mimetypes -import boto3 -import threading +import os +import re import shutil +import sys +import threading +from datetime import datetime, timedelta + +import boto3 from boto3.s3.transfer import S3Transfer + from newsblur_web import settings logger = logging.getLogger(__name__) diff --git a/utils/backups/backup_psql.py b/utils/backups/backup_psql.py index 6fec0466e..0d91af2d4 100644 --- a/utils/backups/backup_psql.py +++ b/utils/backups/backup_psql.py @@ -1,7 +1,8 @@ #!/usr/bin/python3 import os -import sys import socket +import sys + CURRENT_DIR = os.path.dirname(__file__) NEWSBLUR_DIR = ''.join([CURRENT_DIR, '/../../']) sys.path.insert(0, NEWSBLUR_DIR) @@ -9,6 +10,7 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'newsblur_web.settings' import threading + class ProgressPercentage(object): def __init__(self, filename): @@ -29,6 +31,7 @@ class ProgressPercentage(object): sys.stdout.flush() import time + import boto3 from django.conf import settings diff --git a/utils/backups/backup_redis.py b/utils/backups/backup_redis.py index bf8954a9b..b416b0c07 100644 --- a/utils/backups/backup_redis.py +++ b/utils/backups/backup_redis.py @@ -1,7 +1,8 @@ #!/usr/bin/python3 import os -import sys import socket +import sys + CURRENT_DIR = os.path.dirname(__file__) NEWSBLUR_DIR = ''.join([CURRENT_DIR, '/../../']) sys.path.insert(0, NEWSBLUR_DIR) @@ -9,6 +10,7 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'newsblur_web.settings' import threading + class ProgressPercentage(object): def __init__(self, filename): @@ -29,6 +31,7 @@ class ProgressPercentage(object): sys.stdout.flush() import time + import boto3 from django.conf import settings diff --git a/utils/backups/copy_mongo_serialized.py b/utils/backups/copy_mongo_serialized.py index 2e081a5d5..ec773fbc7 100644 --- a/utils/backups/copy_mongo_serialized.py +++ b/utils/backups/copy_mongo_serialized.py @@ -2,9 +2,11 @@ # to another using only pymongo. This circumvents the mongod --repair # option, which can fail. -import sys -import pymongo import datetime +import sys + +import pymongo + from apps.rss_feeds.models import Feed collections = [ diff --git a/utils/exception_middleware.py b/utils/exception_middleware.py index 5eec95079..715b618ad 100644 --- a/utils/exception_middleware.py +++ b/utils/exception_middleware.py @@ -1,6 +1,6 @@ -import traceback -import sys import inspect +import sys +import traceback from pprint import pprint diff --git a/utils/facebook_fetcher.py b/utils/facebook_fetcher.py index 798bd683e..80ed75fc9 100644 --- a/utils/facebook_fetcher.py +++ b/utils/facebook_fetcher.py @@ -1,11 +1,13 @@ -import re import datetime +import re + import dateutil.parser from django.conf import settings from django.utils import feedgenerator from django.utils.html import linebreaks -from apps.social.models import MSocialServices + from apps.reader.models import UserSubscription +from apps.social.models import MSocialServices from utils import log as logging from vendor.facebook import GraphAPIError diff --git a/utils/feed_functions.py b/utils/feed_functions.py index fbd93035e..74e84b7d5 100644 --- a/utils/feed_functions.py +++ b/utils/feed_functions.py @@ -1,14 +1,17 @@ import datetime -import threading -import sys -import traceback import pprint -import urllib.request, urllib.parse, urllib.error -import urllib.parse import random +import sys +import threading +import traceback +import urllib.error +import urllib.parse +import urllib.request import warnings -from django.utils.translation import ungettext + from django.utils.encoding import smart_str +from django.utils.translation import ungettext + from utils import log as logging @@ -235,11 +238,12 @@ def mail_feed_error_to_admin(feed, e, local_vars=None, subject=None): logging.debug(f" ***> Feed error, {subject}: {message}") -## {{{ http://code.activestate.com/recipes/576611/ (r11) -from operator import itemgetter from heapq import nlargest from itertools import repeat +## {{{ http://code.activestate.com/recipes/576611/ (r11) +from operator import itemgetter + class Counter(dict): """Dict subclass for counting hashable objects. Sometimes called a bag diff --git a/utils/feedfinder_forman.py b/utils/feedfinder_forman.py index 8bccbe1bd..9ee7b34ee 100755 --- a/utils/feedfinder_forman.py +++ b/utils/feedfinder_forman.py @@ -13,6 +13,7 @@ if not __FEEDFINDER2_SETUP__: __all__ = ["find_feeds"] import logging + import requests from bs4 import BeautifulSoup from six.moves.urllib import parse as urlparse diff --git a/utils/feedfinder_pilgrim.py b/utils/feedfinder_pilgrim.py index 042f34ec9..d6f57ef67 100644 --- a/utils/feedfinder_pilgrim.py +++ b/utils/feedfinder_pilgrim.py @@ -46,11 +46,17 @@ Also Jason Diamond, Brian Lalor for bug reporting and patches""" _debug = 0 -import sgmllib, urllib.request, urllib.parse, urllib.error, urllib.parse, re, sys, urllib.robotparser -import requests +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +import urllib.robotparser from io import StringIO -from lxml import etree +import requests +import sgmllib +from lxml import etree # XML-RPC support allows feedfinder to query Syndic8 for possible matches. # Python 2.3 now comes with this module by default, otherwise you can download it diff --git a/utils/fields.py b/utils/fields.py index eeba0912f..80ca8d79a 100644 --- a/utils/fields.py +++ b/utils/fields.py @@ -1,13 +1,13 @@ import django -from django.db.transaction import atomic from django.db.models import OneToOneField +from django.db.transaction import atomic try: - from django.db.models.fields.related_descriptors import ( - ReverseOneToOneDescriptor, - ) + from django.db.models.fields.related_descriptors import ReverseOneToOneDescriptor except ImportError: - from django.db.models.fields.related import SingleRelatedObjectDescriptor as ReverseOneToOneDescriptor + from django.db.models.fields.related import ( + SingleRelatedObjectDescriptor as ReverseOneToOneDescriptor, + ) class AutoSingleRelatedObjectDescriptor(ReverseOneToOneDescriptor): diff --git a/utils/grafana_backup.py b/utils/grafana_backup.py index 9dd933da3..eb725c7e8 100644 --- a/utils/grafana_backup.py +++ b/utils/grafana_backup.py @@ -1,4 +1,5 @@ import json + import requests with open("/srv/secrets-newsblur/keys/grafana_api_key", "r") as file: diff --git a/utils/hostname_ssh.py b/utils/hostname_ssh.py index 30eea4a93..69138a701 100644 --- a/utils/hostname_ssh.py +++ b/utils/hostname_ssh.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -import sys +import json import os import subprocess -import json +import sys + import digitalocean # from django.conf import settings diff --git a/utils/image_functions.py b/utils/image_functions.py index fb5448c73..91d222518 100644 --- a/utils/image_functions.py +++ b/utils/image_functions.py @@ -1,11 +1,11 @@ """Operations for images through the PIL.""" import urllib.request -from PIL import Image -from PIL import ImageFile +from io import BytesIO + +from PIL import Image, ImageFile from PIL import ImageOps as PILOps from PIL.ExifTags import TAGS -from io import BytesIO PROFILE_PICTURE_SIZES = {"fullsize": (256, 256), "thumbnail": (64, 64)} diff --git a/utils/json_fetcher.py b/utils/json_fetcher.py index d94371c32..09c6c0c55 100644 --- a/utils/json_fetcher.py +++ b/utils/json_fetcher.py @@ -1,7 +1,9 @@ import datetime + import dateutil.parser from django.conf import settings from django.utils import feedgenerator + from utils import log as logging from utils.json_functions import decode diff --git a/utils/json_functions.py b/utils/json_functions.py index 0cf5fa4e3..feb349bb6 100644 --- a/utils/json_functions.py +++ b/utils/json_functions.py @@ -1,20 +1,22 @@ # -*- coding: utf-8 -*- -from django.db import models -from django.utils.functional import Promise -from django.utils.encoding import force_text, smart_str +import datetime import json +import sys from decimal import Decimal -from django.core import serializers + +from bson.objectid import ObjectId from django.conf import settings -from django.http import HttpResponse, HttpResponseForbidden, Http404 +from django.core import serializers +from django.db import models from django.db.models.query import QuerySet +from django.http import Http404, HttpResponse, HttpResponseForbidden +from django.utils.encoding import force_text, smart_str +from django.utils.functional import Promise # from django.utils.deprecation import CallableBool from mongoengine.queryset.queryset import QuerySet as MongoQuerySet -from bson.objectid import ObjectId + from utils import log as logging -import sys -import datetime def decode(data): diff --git a/utils/log.py b/utils/log.py index de81fb008..987ebd4b3 100644 --- a/utils/log.py +++ b/utils/log.py @@ -3,9 +3,9 @@ import re import string import time -from django.core.handlers.wsgi import WSGIRequest from django.conf import settings -from django.utils.encoding import smart_str, smart_str +from django.core.handlers.wsgi import WSGIRequest +from django.utils.encoding import smart_str from .user_functions import extract_user_agent diff --git a/utils/management_functions.py b/utils/management_functions.py index e49c231f4..dceec56ef 100644 --- a/utils/management_functions.py +++ b/utils/management_functions.py @@ -1,5 +1,5 @@ -import os import errno +import os def daemonize(): diff --git a/utils/mongo_command_monitor.py b/utils/mongo_command_monitor.py index 108fcfe99..2ff21cd91 100644 --- a/utils/mongo_command_monitor.py +++ b/utils/mongo_command_monitor.py @@ -1,7 +1,8 @@ -from pymongo import monitoring import logging + from django.conf import settings from django.db import connection +from pymongo import monitoring class MongoCommandLogger(monitoring.CommandListener): diff --git a/utils/mongo_raw_log_middleware.py b/utils/mongo_raw_log_middleware.py index 5d23e668d..e8c5536ed 100644 --- a/utils/mongo_raw_log_middleware.py +++ b/utils/mongo_raw_log_middleware.py @@ -1,14 +1,16 @@ -from django.core.exceptions import MiddlewareNotUsed -from django.conf import settings -from django.db import connection -from pymongo.mongo_client import MongoClient -from pymongo.mongo_replica_set_client import MongoReplicaSetClient -from time import time -from utils import log as logging import struct +from time import time + import bson import pymongo from bson.errors import InvalidBSON +from django.conf import settings +from django.core.exceptions import MiddlewareNotUsed +from django.db import connection +from pymongo.mongo_client import MongoClient +from pymongo.mongo_replica_set_client import MongoReplicaSetClient + +from utils import log as logging class MongoDumpMiddleware(object): diff --git a/utils/mongoengine_fields.py b/utils/mongoengine_fields.py index 9209ec91a..dff08d902 100644 --- a/utils/mongoengine_fields.py +++ b/utils/mongoengine_fields.py @@ -1,6 +1,7 @@ from datetime import timedelta + from mongoengine.base import BaseField -from mongoengine.fields import IntField, StringField, EmailField +from mongoengine.fields import EmailField, IntField, StringField class TimedeltaField(BaseField): diff --git a/utils/monitor_disk_usage.py b/utils/monitor_disk_usage.py index b9d581e8c..8b8ff44ee 100755 --- a/utils/monitor_disk_usage.py +++ b/utils/monitor_disk_usage.py @@ -3,10 +3,12 @@ import sys sys.path.append("/srv/newsblur") -import requests -from newsblur_web import settings import socket +import requests + +from newsblur_web import settings + def main(): disk_usage_output = sys.argv[1] diff --git a/utils/monitor_newsletter_delivery.py b/utils/monitor_newsletter_delivery.py index c1ef3a57c..253c4270f 100755 --- a/utils/monitor_newsletter_delivery.py +++ b/utils/monitor_newsletter_delivery.py @@ -4,10 +4,12 @@ import sys sys.path.append("/srv/newsblur") -import requests -from newsblur_web import settings import socket +import requests + +from newsblur_web import settings + def main(): hostname = socket.gethostname() diff --git a/utils/monitor_redis_bgsave.py b/utils/monitor_redis_bgsave.py index c23769ba1..fda9df230 100755 --- a/utils/monitor_redis_bgsave.py +++ b/utils/monitor_redis_bgsave.py @@ -4,12 +4,14 @@ import sys sys.path.append("/srv/newsblur") -import os import datetime -import requests -from newsblur_web import settings +import os import socket +import requests + +from newsblur_web import settings + def main(): redis_log_path = sys.argv[1] diff --git a/utils/monitor_task_fetches.py b/utils/monitor_task_fetches.py index 56e4d967e..4e4ae98c0 100755 --- a/utils/monitor_task_fetches.py +++ b/utils/monitor_task_fetches.py @@ -4,11 +4,13 @@ import sys sys.path.append("/srv/newsblur") -import requests -from newsblur_web import settings import socket -import redis + import pymongo +import redis +import requests + +from newsblur_web import settings def main(): diff --git a/utils/monitor_work_queue.py b/utils/monitor_work_queue.py index df95cb1f2..6fc4b4673 100755 --- a/utils/monitor_work_queue.py +++ b/utils/monitor_work_queue.py @@ -4,11 +4,13 @@ import sys sys.path.append("/srv/newsblur") -import requests -from newsblur_web import settings import socket -import redis + import pymongo +import redis +import requests + +from newsblur_web import settings def main(): diff --git a/utils/munin/newsblur_app_servers.py b/utils/munin/newsblur_app_servers.py index d4b1766ea..04e05a02f 100755 --- a/utils/munin/newsblur_app_servers.py +++ b/utils/munin/newsblur_app_servers.py @@ -1,8 +1,9 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import datetime import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" from django.conf import settings @@ -54,6 +55,7 @@ class NBMuninGraph(MuninGraph): @property def total(self): import datetime + from django.conf import settings stats = settings.MONGOANALYTICSDB.nbanalytics.page_loads.aggregate( diff --git a/utils/munin/newsblur_classifiers.py b/utils/munin/newsblur_classifiers.py index 1efbd6e9a..3e28bf8e9 100755 --- a/utils/munin/newsblur_classifiers.py +++ b/utils/munin/newsblur_classifiers.py @@ -1,8 +1,9 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django @@ -24,7 +25,12 @@ class NBMuninGraph(MuninGraph): } def calculate_metrics(self): - from apps.analyzer.models import MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle + from apps.analyzer.models import ( + MClassifierAuthor, + MClassifierFeed, + MClassifierTag, + MClassifierTitle, + ) return { "feeds": MClassifierFeed.objects.count(), diff --git a/utils/munin/newsblur_dbtimes.py b/utils/munin/newsblur_dbtimes.py index e50be6597..740faeb26 100755 --- a/utils/munin/newsblur_dbtimes.py +++ b/utils/munin/newsblur_dbtimes.py @@ -1,7 +1,8 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django diff --git a/utils/munin/newsblur_errors.py b/utils/munin/newsblur_errors.py index ec3c967e3..9c040d356 100755 --- a/utils/munin/newsblur_errors.py +++ b/utils/munin/newsblur_errors.py @@ -1,8 +1,9 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django diff --git a/utils/munin/newsblur_feed_counts.py b/utils/munin/newsblur_feed_counts.py index 97d929802..3c8e13938 100755 --- a/utils/munin/newsblur_feed_counts.py +++ b/utils/munin/newsblur_feed_counts.py @@ -1,8 +1,10 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph -import redis import os +import redis + +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django @@ -26,9 +28,10 @@ class NBMuninGraph(MuninGraph): } def calculate_metrics(self): - from apps.rss_feeds.models import Feed, DuplicateFeed - from apps.push.models import PushSubscription from django.conf import settings + + from apps.push.models import PushSubscription + from apps.rss_feeds.models import DuplicateFeed, Feed from apps.statistics.models import MStatistics exception_feeds = MStatistics.get("munin:exception_feeds") diff --git a/utils/munin/newsblur_feeds.py b/utils/munin/newsblur_feeds.py index c02918e6f..fd14df255 100755 --- a/utils/munin/newsblur_feeds.py +++ b/utils/munin/newsblur_feeds.py @@ -1,7 +1,8 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django @@ -23,8 +24,8 @@ class NBMuninGraph(MuninGraph): } def calculate_metrics(self): - from apps.rss_feeds.models import Feed from apps.reader.models import UserSubscription + from apps.rss_feeds.models import Feed from apps.social.models import MSocialProfile, MSocialSubscription from apps.statistics.models import MStatistics diff --git a/utils/munin/newsblur_loadtimes.py b/utils/munin/newsblur_loadtimes.py index 5d89be785..4f17a32d1 100755 --- a/utils/munin/newsblur_loadtimes.py +++ b/utils/munin/newsblur_loadtimes.py @@ -1,7 +1,8 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django diff --git a/utils/munin/newsblur_stories.py b/utils/munin/newsblur_stories.py index d94c35d2c..c035ce77f 100755 --- a/utils/munin/newsblur_stories.py +++ b/utils/munin/newsblur_stories.py @@ -1,7 +1,8 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django @@ -21,7 +22,7 @@ class NBMuninGraph(MuninGraph): } def calculate_metrics(self): - from apps.rss_feeds.models import MStory, MStarredStory + from apps.rss_feeds.models import MStarredStory, MStory return { "stories": MStory.objects.count(), diff --git a/utils/munin/newsblur_tasks_codes.py b/utils/munin/newsblur_tasks_codes.py index e613704aa..a79d53f8f 100755 --- a/utils/munin/newsblur_tasks_codes.py +++ b/utils/munin/newsblur_tasks_codes.py @@ -1,7 +1,8 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" @@ -28,6 +29,7 @@ class NBMuninGraph(MuninGraph): @property def stats(self): import datetime + from django.conf import settings stats = settings.MONGOANALYTICSDB.nbanalytics.feed_fetches.aggregate( diff --git a/utils/munin/newsblur_tasks_pipeline.py b/utils/munin/newsblur_tasks_pipeline.py index f857452d7..36c071faf 100755 --- a/utils/munin/newsblur_tasks_pipeline.py +++ b/utils/munin/newsblur_tasks_pipeline.py @@ -1,7 +1,8 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django @@ -30,6 +31,7 @@ class NBMuninGraph(MuninGraph): @property def stats(self): import datetime + from django.conf import settings stats = settings.MONGOANALYTICSDB.nbanalytics.feed_fetches.aggregate( diff --git a/utils/munin/newsblur_tasks_servers.py b/utils/munin/newsblur_tasks_servers.py index e76b920dc..b3cc597ca 100755 --- a/utils/munin/newsblur_tasks_servers.py +++ b/utils/munin/newsblur_tasks_servers.py @@ -1,8 +1,9 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import datetime import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" from django.conf import settings @@ -54,6 +55,7 @@ class NBMuninGraph(MuninGraph): @property def total(self): import datetime + from django.conf import settings stats = settings.MONGOANALYTICSDB.nbanalytics.feed_fetches.aggregate( diff --git a/utils/munin/newsblur_tasks_times.py b/utils/munin/newsblur_tasks_times.py index e83d29ff8..44bcc1176 100755 --- a/utils/munin/newsblur_tasks_times.py +++ b/utils/munin/newsblur_tasks_times.py @@ -1,7 +1,8 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django @@ -33,6 +34,7 @@ class NBMuninGraph(MuninGraph): @property def stats(self): import datetime + from django.conf import settings stats = settings.MONGOANALYTICSDB.nbanalytics.feed_fetches.aggregate( diff --git a/utils/munin/newsblur_updates.py b/utils/munin/newsblur_updates.py index 4be495c4f..5d57fd972 100755 --- a/utils/munin/newsblur_updates.py +++ b/utils/munin/newsblur_updates.py @@ -1,8 +1,10 @@ #!/srv/newsblur/venv/newsblur3/bin/python -import redis -from utils.munin.base import MuninGraph import os +import redis + +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django diff --git a/utils/munin/newsblur_users.py b/utils/munin/newsblur_users.py index 5725dcfcc..2a0048e7f 100755 --- a/utils/munin/newsblur_users.py +++ b/utils/munin/newsblur_users.py @@ -1,7 +1,8 @@ #!/srv/newsblur/venv/newsblur3/bin/python -from utils.munin.base import MuninGraph import os +from utils.munin.base import MuninGraph + os.environ["DJANGO_SETTINGS_MODULE"] = "newsblur_web.settings" import django @@ -25,7 +26,9 @@ class NBMuninGraph(MuninGraph): def calculate_metrics(self): import datetime + from django.contrib.auth.models import User + from apps.profile.models import Profile, RNewUserQueue last_month = datetime.datetime.utcnow() - datetime.timedelta(days=30) diff --git a/utils/pipeline_utils.py b/utils/pipeline_utils.py index a73f5c4bf..c400e3b92 100644 --- a/utils/pipeline_utils.py +++ b/utils/pipeline_utils.py @@ -1,9 +1,9 @@ import re + from django.conf import settings -from pipeline.finders import FileSystemFinder as PipelineFileSystemFinder from pipeline.finders import AppDirectoriesFinder as PipelineAppDirectoriesFinder -from pipeline.storage import GZIPMixin -from pipeline.storage import PipelineManifestStorage +from pipeline.finders import FileSystemFinder as PipelineFileSystemFinder +from pipeline.storage import GZIPMixin, PipelineManifestStorage class PipelineStorage(PipelineManifestStorage): diff --git a/utils/ratelimit.py b/utils/ratelimit.py index 9e8293324..fe5824e03 100644 --- a/utils/ratelimit.py +++ b/utils/ratelimit.py @@ -1,8 +1,9 @@ -from django.http import HttpResponse -from django.core.cache import cache -from datetime import datetime, timedelta import functools import hashlib +from datetime import datetime, timedelta + +from django.core.cache import cache +from django.http import HttpResponse class ratelimit(object): diff --git a/utils/redis_raw_log_middleware.py b/utils/redis_raw_log_middleware.py index 6e11bc13e..e309bc8e1 100644 --- a/utils/redis_raw_log_middleware.py +++ b/utils/redis_raw_log_middleware.py @@ -1,10 +1,11 @@ -from django.core.exceptions import MiddlewareNotUsed -from django.conf import settings -from django.db import connection -from redis.connection import Connection -from redis.client import Redis, Pipeline -from time import time from pprint import pprint +from time import time + +from django.conf import settings +from django.core.exceptions import MiddlewareNotUsed +from django.db import connection +from redis.client import Pipeline, Redis +from redis.connection import Connection class RedisDumpMiddleware(object): diff --git a/utils/request_introspection_middleware.py b/utils/request_introspection_middleware.py index 21e8a594e..7fa53f4f8 100644 --- a/utils/request_introspection_middleware.py +++ b/utils/request_introspection_middleware.py @@ -1,10 +1,12 @@ -from django.conf import settings -from utils import log as logging -from apps.statistics.rstats import round_time -import pickle import base64 +import pickle import time + import redis +from django.conf import settings + +from apps.statistics.rstats import round_time +from utils import log as logging IGNORE_PATHS = [ "/_haproxychk", diff --git a/utils/s3_utils.py b/utils/s3_utils.py index f71ec07a7..43a9d7bd0 100644 --- a/utils/s3_utils.py +++ b/utils/s3_utils.py @@ -1,7 +1,8 @@ +import mimetypes import os import sys import time -import mimetypes + from utils.image_functions import ImageOps if "/srv/newsblur" not in " ".join(sys.path): diff --git a/utils/scrubber/__init__.py b/utils/scrubber/__init__.py index f39114286..d60dbc2f2 100755 --- a/utils/scrubber/__init__.py +++ b/utils/scrubber/__init__.py @@ -11,9 +11,11 @@ __version__ = "1.6.1" __license__ = "BSD" __all__ = ["Scrubber", "SelectiveScriptScrubber", "ScrubberWarning", "UnapprovedJavascript", "urlize"] -import re, string -from urllib.parse import urljoin +import re +import string from itertools import chain +from urllib.parse import urljoin + from bs4 import BeautifulSoup, Comment diff --git a/utils/story_functions.py b/utils/story_functions.py index dc5092f3e..171a2301a 100644 --- a/utils/story_functions.py +++ b/utils/story_functions.py @@ -1,26 +1,28 @@ -import re -import datetime -import struct -import dateutil -import hashlib import base64 -import html -import sys -from random import randint -from lxml.html.diff import tokenize, fixup_ins_del_tags, htmldiff_tokens -from lxml.etree import ParserError, XMLSyntaxError, SerialisationError -import lxml.html, lxml.etree -from lxml.html.clean import Cleaner -from itertools import chain -from django.utils.dateformat import DateFormat -from django.utils.html import strip_tags as strip_tags_django -from utils.tornado_escape import linkify as linkify_tornado -from utils.tornado_escape import xhtml_unescape as xhtml_unescape_tornado -import feedparser - +import datetime +import hashlib import hmac +import html +import re +import struct +import sys from binascii import hexlify from hashlib import sha1 +from itertools import chain +from random import randint + +import dateutil +import feedparser +import lxml.etree +import lxml.html +from django.utils.dateformat import DateFormat +from django.utils.html import strip_tags as strip_tags_django +from lxml.etree import ParserError, SerialisationError, XMLSyntaxError +from lxml.html.clean import Cleaner +from lxml.html.diff import fixup_ins_del_tags, htmldiff_tokens, tokenize + +from utils.tornado_escape import linkify as linkify_tornado +from utils.tornado_escape import xhtml_unescape as xhtml_unescape_tornado # COMMENTS_RE = re.compile('\') COMMENTS_RE = re.compile("\