import base64
import concurrent
import datetime
import random
import re
import socket
import ssl
import time
import urllib.error
import urllib.parse
import urllib.request
import zlib
import redis
import requests
from django.conf import settings
from django.contrib.auth import login as login_user
from django.contrib.auth import logout as logout_user
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.core.mail import EmailMultiAlternatives
from django.core.validators import validate_email
from django.db import IntegrityError
from django.db.models import Q
from django.db.utils import DatabaseError
from django.http import (
Http404,
HttpResponse,
HttpResponseForbidden,
HttpResponseRedirect,
UnreadablePostError,
)
from django.shortcuts import get_object_or_404, render
from django.template.loader import render_to_string
from django.urls import reverse
from django.utils import feedgenerator
from django.utils.encoding import smart_str
from django.views.decorators.cache import never_cache
from mongoengine.queryset import NotUniqueError, OperationError
from apps.analyzer.models import (
MClassifierAuthor,
MClassifierFeed,
MClassifierTag,
MClassifierTitle,
apply_classifier_authors,
apply_classifier_feeds,
apply_classifier_tags,
apply_classifier_titles,
get_classifiers_for_user,
sort_classifiers_by_feed,
)
from apps.notifications.models import MUserFeedNotification
from apps.profile.models import MCustomStyling, MDashboardRiver, Profile
from apps.reader.forms import FeatureForm, LoginForm, SignupForm
from apps.reader.models import (
Feature,
RUserStory,
RUserUnreadStory,
UserSubscription,
UserSubscriptionFolders,
)
from apps.recommendations.models import RecommendedFeed
from apps.rss_feeds.models import MFeedIcon, MSavedSearch, MStarredStoryCounts
from apps.search.models import MUserSearch
from apps.statistics.models import MAnalyticsLoader, MStatistics
from apps.statistics.rstats import RStats
# from apps.search.models import SearchStarredStory
try:
from apps.rss_feeds.models import (
DuplicateFeed,
Feed,
MFeedPage,
MStarredStory,
MStory,
)
except:
pass
import tweepy
from apps.categories.models import MCategory
from apps.rss_feeds.tasks import ScheduleImmediateFetches
from apps.social.models import (
MActivity,
MInteraction,
MSharedStory,
MSocialProfile,
MSocialServices,
MSocialSubscription,
)
from apps.social.views import load_social_page
from utils import json_functions as json
from utils import log as logging
from utils.feed_functions import relative_timesince
from utils.ratelimit import ratelimit
from utils.story_functions import (
format_story_link_date__long,
format_story_link_date__short,
strip_tags,
)
from utils.user_functions import ajax_login_required, extract_user_agent, get_user
from utils.view_functions import (
get_argument_or_404,
is_true,
render_to,
required_params,
)
from vendor.timezones.utilities import localtime_for_timezone
BANNED_URLS = [
"brentozar.com",
]
ALLOWED_SUBDOMAINS = [
'dev',
'www',
'hwww',
'dwww',
# 'beta', # Comment to redirect beta -> www, uncomment to allow beta -> staging (+ dns changes)
'staging',
'hstaging',
'discovery',
'debug',
'debug3',
'staging2',
'staging3',
'nb',
]
def get_subdomain(request):
host = request.META.get('HTTP_HOST')
if host and host.count(".") >= 2:
return host.split(".")[0]
else:
return None
@never_cache
@render_to('reader/dashboard.xhtml')
def index(request, **kwargs):
subdomain = get_subdomain(request)
if request.method == "GET" and subdomain and subdomain not in ALLOWED_SUBDOMAINS:
username = request.subdomain or subdomain
if '.' in username:
username = username.split('.')[0]
user = User.objects.filter(username=username)
if not user:
user = User.objects.filter(username__iexact=username)
if user:
user = user[0]
if not user:
return HttpResponseRedirect('http://%s%s' % (
Site.objects.get_current().domain,
reverse('index')))
return load_social_page(request, user_id=user.pk, username=request.subdomain, **kwargs)
if request.user.is_anonymous:
return welcome(request, **kwargs)
else:
return dashboard(request, **kwargs)
def dashboard(request, **kwargs):
user = request.user
feed_count = UserSubscription.objects.filter(user=request.user).count()
# recommended_feeds = RecommendedFeed.objects.filter(is_public=True,
# approved_date__lte=datetime.datetime.now()
# ).select_related('feed')[:2]
unmoderated_feeds = []
if user.is_staff:
unmoderated_feeds = RecommendedFeed.objects.filter(is_public=False,
declined_date__isnull=True
).select_related('feed')[:2]
statistics = MStatistics.all()
social_profile = MSocialProfile.get_user(user.pk)
custom_styling = MCustomStyling.get_user(user.pk)
dashboard_rivers = MDashboardRiver.get_user_rivers(user.pk)
preferences = json.decode(user.profile.preferences)
if not user.is_active:
url = "https://%s%s" % (Site.objects.get_current().domain,
reverse('stripe-form'))
return HttpResponseRedirect(url)
logging.user(request, "~FBLoading dashboard")
return {
'user_profile' : user.profile,
'preferences' : preferences,
'feed_count' : feed_count,
'custom_styling' : custom_styling,
'dashboard_rivers' : dashboard_rivers,
'account_images' : list(range(1, 4)),
# 'recommended_feeds' : recommended_feeds,
'unmoderated_feeds' : unmoderated_feeds,
'statistics' : statistics,
'social_profile' : social_profile,
'debug' : settings.DEBUG,
'debug_assets' : settings.DEBUG_ASSETS,
}, "reader/dashboard.xhtml"
@render_to('reader/dashboard.xhtml')
def welcome_req(request, **kwargs):
return welcome(request, **kwargs)
def welcome(request, **kwargs):
user = get_user(request)
statistics = MStatistics.all()
social_profile = MSocialProfile.get_user(user.pk)
if request.method == "POST":
if request.POST.get('submit', '').startswith('log'):
login_form = LoginForm(request.POST, prefix='login')
signup_form = SignupForm(prefix='signup')
else:
signup_form = SignupForm(request.POST, prefix='signup')
return {
"form": signup_form
}, "accounts/signup.html"
else:
login_form = LoginForm(prefix='login')
signup_form = SignupForm(prefix='signup')
logging.user(request, "~FBLoading welcome")
return {
'user_profile' : hasattr(user, 'profile') and user.profile,
'login_form' : login_form,
'signup_form' : signup_form,
'statistics' : statistics,
'social_profile' : social_profile,
'post_request' : request.method == 'POST',
}, "reader/welcome.xhtml"
@never_cache
def login(request):
code = -1
message = ""
if request.method == "POST":
form = LoginForm(request.POST, prefix='login')
if form.is_valid():
login_user(request, form.get_user(), backend='django.contrib.auth.backends.ModelBackend')
if request.POST.get('api'):
logging.user(form.get_user(), "~FG~BB~SKiPhone Login~FW")
code = 1
else:
logging.user(form.get_user(), "~FG~BBLogin~FW")
next_url = request.POST.get('next', '')
if next_url:
return HttpResponseRedirect(next_url)
return HttpResponseRedirect(reverse('index'))
else:
message = list(form.errors.items())[0][1][0]
if request.POST.get('api'):
return HttpResponse(json.encode(dict(code=code, message=message)), content_type='application/json')
else:
return index(request)
@never_cache
@render_to('accounts/signup.html')
def signup(request):
if request.method == "POST":
if settings.ENFORCE_SIGNUP_CAPTCHA:
signup_form = SignupForm(request.POST, prefix='signup')
return {
"form": signup_form
}
form = SignupForm(prefix='signup', data=request.POST)
if form.is_valid():
new_user = form.save()
login_user(request, new_user, backend='django.contrib.auth.backends.ModelBackend')
logging.user(new_user, "~FG~SB~BBNEW SIGNUP: ~FW%s" % new_user.email)
if not new_user.is_active:
url = "https://%s%s" % (Site.objects.get_current().domain,
reverse('stripe-form'))
return HttpResponseRedirect(url)
else:
return HttpResponseRedirect(reverse('index'))
return index(request)
@never_cache
def logout(request):
logging.user(request, "~FG~BBLogout~FW")
logout_user(request)
if request.GET.get('api'):
return HttpResponse(json.encode(dict(code=1)), content_type='application/json')
else:
return HttpResponseRedirect(reverse('index'))
def autologin(request, username, secret):
next = request.GET.get('next', '')
if not username or not secret:
return HttpResponseForbidden()
profile = Profile.objects.filter(user__username=username, secret_token=secret)
if not profile:
return HttpResponseForbidden()
user = profile[0].user
user.backend = settings.AUTHENTICATION_BACKENDS[0]
login_user(request, user, backend='django.contrib.auth.backends.ModelBackend')
logging.user(user, "~FG~BB~SKAuto-Login. Next stop: %s~FW" % (next if next else 'Homepage',))
if next and not next.startswith('/'):
next = '?next=' + next
return HttpResponseRedirect(reverse('index') + next)
elif next:
return HttpResponseRedirect(next)
else:
return HttpResponseRedirect(reverse('index'))
@ratelimit(minutes=1, requests=60)
@never_cache
@json.json_view
def load_feeds(request):
user = get_user(request)
feeds = {}
include_favicons = is_true(request.GET.get('include_favicons', False))
flat = is_true(request.GET.get('flat', False))
update_counts = is_true(request.GET.get('update_counts', True))
version = int(request.GET.get('v', 1))
if include_favicons == 'false': include_favicons = False
if update_counts == 'false': update_counts = False
if flat == 'false': flat = False
if flat: return load_feeds_flat(request)
platform = extract_user_agent(request)
if platform in ['iPhone', 'iPad', 'Androd']:
# Remove this check once the iOS and Android updates go out which have update_counts=False
# and then guarantee a refresh_feeds call
update_counts = False
try:
folders = UserSubscriptionFolders.objects.get(user=user)
except UserSubscriptionFolders.DoesNotExist:
data = dict(feeds=[], folders=[])
return data
except UserSubscriptionFolders.MultipleObjectsReturned:
UserSubscriptionFolders.objects.filter(user=user)[1:].delete()
folders = UserSubscriptionFolders.objects.get(user=user)
user_subs = UserSubscription.objects.select_related('feed').filter(user=user)
notifications = MUserFeedNotification.feeds_for_user(user.pk)
day_ago = datetime.datetime.now() - datetime.timedelta(days=1)
scheduled_feeds = []
for sub in user_subs:
pk = sub.feed_id
if update_counts and sub.needs_unread_recalc:
sub.calculate_feed_scores(silent=True)
feeds[pk] = sub.canonical(include_favicon=include_favicons)
if not sub.active: continue
if pk in notifications:
feeds[pk].update(notifications[pk])
if not sub.feed.active and not sub.feed.has_feed_exception:
scheduled_feeds.append(sub.feed.pk)
elif sub.feed.active_subscribers <= 0:
scheduled_feeds.append(sub.feed.pk)
elif sub.feed.next_scheduled_update < day_ago:
scheduled_feeds.append(sub.feed.pk)
if len(scheduled_feeds) > 0 and request.user.is_authenticated:
logging.user(request, "~SN~FMTasking the scheduling immediate fetch of ~SB%s~SN feeds..." %
len(scheduled_feeds))
ScheduleImmediateFetches.apply_async(kwargs=dict(feed_ids=scheduled_feeds, user_id=user.pk))
starred_counts, starred_count = MStarredStoryCounts.user_counts(user.pk, include_total=True)
if not starred_count and len(starred_counts):
starred_count = MStarredStory.objects(user_id=user.pk).count()
saved_searches = MSavedSearch.user_searches(user.pk)
social_params = {
'user_id': user.pk,
'include_favicon': include_favicons,
'update_counts': update_counts,
}
social_feeds = MSocialSubscription.feeds(**social_params)
social_profile = MSocialProfile.profile(user.pk)
social_services = MSocialServices.profile(user.pk)
dashboard_rivers = MDashboardRiver.get_user_rivers(user.pk)
categories = None
if not user_subs:
categories = MCategory.serialize()
logging.user(request, "~FB~SBLoading ~FY%s~FB/~FM%s~FB feeds/socials%s" % (
len(list(feeds.keys())), len(social_feeds), '. ~FCUpdating counts.' if update_counts else ''))
data = {
'feeds': list(feeds.values()) if version == 2 else feeds,
'social_feeds': social_feeds,
'social_profile': social_profile,
'social_services': social_services,
'user_profile': user.profile,
"is_staff": user.is_staff,
'user_id': user.pk,
'folders': json.decode(folders.folders),
'starred_count': starred_count,
'starred_counts': starred_counts,
'saved_searches': saved_searches,
'dashboard_rivers': dashboard_rivers,
'categories': categories,
'share_ext_token': user.profile.secret_token,
}
return data
@json.json_view
def load_feed_favicons(request):
user = get_user(request)
feed_ids = request.GET.getlist('feed_ids') or request.GET.getlist('feed_ids[]')
if not feed_ids:
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
feed_ids = [sub['feed__pk'] for sub in user_subs.values('feed__pk')]
feed_icons = dict([(i.feed_id, i.data) for i in MFeedIcon.objects(feed_id__in=feed_ids)])
return feed_icons
def load_feeds_flat(request):
user = request.user
include_favicons = is_true(request.GET.get('include_favicons', False))
update_counts = is_true(request.GET.get('update_counts', True))
include_inactive = is_true(request.GET.get('include_inactive', False))
background_ios = is_true(request.GET.get('background_ios', False))
feeds = {}
inactive_feeds = {}
day_ago = datetime.datetime.now() - datetime.timedelta(days=1)
scheduled_feeds = []
iphone_version = "2.1" # Preserved forever. Don't change.
latest_ios_build = "52"
latest_ios_version = "5.0.0b2"
if include_favicons == 'false': include_favicons = False
if update_counts == 'false': update_counts = False
if not user.is_authenticated:
return HttpResponseForbidden()
try:
folders = UserSubscriptionFolders.objects.get(user=user)
except UserSubscriptionFolders.DoesNotExist:
folders = []
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
notifications = MUserFeedNotification.feeds_for_user(user.pk)
if not user_subs and folders:
folders.auto_activate()
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
if include_inactive:
inactive_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=False)
for sub in user_subs:
pk = sub.feed_id
if update_counts and sub.needs_unread_recalc:
sub.calculate_feed_scores(silent=True)
feeds[pk] = sub.canonical(include_favicon=include_favicons)
if not sub.feed.active and not sub.feed.has_feed_exception:
scheduled_feeds.append(sub.feed.pk)
elif sub.feed.active_subscribers <= 0:
scheduled_feeds.append(sub.feed.pk)
elif sub.feed.next_scheduled_update < day_ago:
scheduled_feeds.append(sub.feed.pk)
if pk in notifications:
feeds[pk].update(notifications[pk])
if include_inactive:
for sub in inactive_subs:
inactive_feeds[sub.feed_id] = sub.canonical(include_favicon=include_favicons)
if len(scheduled_feeds) > 0 and request.user.is_authenticated:
logging.user(request, "~SN~FMTasking the scheduling immediate fetch of ~SB%s~SN feeds..." %
len(scheduled_feeds))
ScheduleImmediateFetches.apply_async(kwargs=dict(feed_ids=scheduled_feeds, user_id=user.pk))
flat_folders = []
flat_folders_with_inactive = []
if folders:
flat_folders = folders.flatten_folders(feeds=feeds)
flat_folders_with_inactive = folders.flatten_folders(feeds=feeds,
inactive_feeds=inactive_feeds)
social_params = {
'user_id': user.pk,
'include_favicon': include_favicons,
'update_counts': update_counts,
}
social_feeds = MSocialSubscription.feeds(**social_params)
social_profile = MSocialProfile.profile(user.pk)
social_services = MSocialServices.profile(user.pk)
starred_counts, starred_count = MStarredStoryCounts.user_counts(user.pk, include_total=True)
if not starred_count and len(starred_counts):
starred_count = MStarredStory.objects(user_id=user.pk).count()
categories = None
if not user_subs:
categories = MCategory.serialize()
saved_searches = MSavedSearch.user_searches(user.pk)
logging.user(request, "~FB~SBLoading ~FY%s~FB/~FM%s~FB/~FR%s~FB feeds/socials/inactive ~FMflat~FB%s%s" % (
len(list(feeds.keys())), len(social_feeds), len(inactive_feeds), '. ~FCUpdating counts.' if update_counts else '',
' ~BB(background fetch)' if background_ios else ''))
data = {
"flat_folders": flat_folders,
"flat_folders_with_inactive": flat_folders_with_inactive,
"feeds": feeds,
"inactive_feeds": inactive_feeds if include_inactive else {"0": "Include `include_inactive=true`"},
"social_feeds": social_feeds,
"social_profile": social_profile,
"social_services": social_services,
"user": user.username,
"user_id": user.pk,
"is_staff": user.is_staff,
"user_profile": user.profile,
"iphone_version": iphone_version,
"latest_ios_build": latest_ios_build,
"latest_ios_version": latest_ios_version,
"categories": categories,
'starred_count': starred_count,
'starred_counts': starred_counts,
'saved_searches': saved_searches,
'share_ext_token': user.profile.secret_token,
}
return data
class ratelimit_refresh_feeds(ratelimit):
def should_ratelimit(self, request):
feed_ids = request.POST.getlist('feed_id') or request.POST.getlist('feed_id[]')
if len(feed_ids) == 1:
return False
return True
@ratelimit_refresh_feeds(minutes=1, requests=30)
@never_cache
@json.json_view
def refresh_feeds(request):
get_post = getattr(request, request.method)
start = datetime.datetime.now()
start_time = time.time()
user = get_user(request)
feed_ids = get_post.getlist('feed_id') or get_post.getlist('feed_id[]')
check_fetch_status = get_post.get('check_fetch_status')
favicons_fetching = get_post.getlist('favicons_fetching') or get_post.getlist('favicons_fetching[]')
social_feed_ids = [feed_id for feed_id in feed_ids if 'social:' in feed_id]
feed_ids = list(set(feed_ids) - set(social_feed_ids))
feeds = {}
if feed_ids or (not social_feed_ids and not feed_ids):
feeds = UserSubscription.feeds_with_updated_counts(user, feed_ids=feed_ids,
check_fetch_status=check_fetch_status)
checkpoint1 = datetime.datetime.now()
social_feeds = {}
if social_feed_ids or (not social_feed_ids and not feed_ids):
social_feeds = MSocialSubscription.feeds_with_updated_counts(user, social_feed_ids=social_feed_ids)
checkpoint2 = datetime.datetime.now()
favicons_fetching = [int(f) for f in favicons_fetching if f]
feed_icons = {}
if favicons_fetching:
feed_icons = dict([(i.feed_id, i) for i in MFeedIcon.objects(feed_id__in=favicons_fetching)])
for feed_id, feed in list(feeds.items()):
if feed_id in favicons_fetching and feed_id in feed_icons:
feeds[feed_id]['favicon'] = feed_icons[feed_id].data
feeds[feed_id]['favicon_color'] = feed_icons[feed_id].color
feeds[feed_id]['favicon_fetching'] = feed.get('favicon_fetching')
user_subs = UserSubscription.objects.filter(user=user, active=True).only('feed')
sub_feed_ids = [s.feed_id for s in user_subs]
if favicons_fetching:
moved_feed_ids = [f for f in favicons_fetching if f not in sub_feed_ids]
for moved_feed_id in moved_feed_ids:
duplicate_feeds = DuplicateFeed.objects.filter(duplicate_feed_id=moved_feed_id)
if duplicate_feeds and duplicate_feeds[0].feed.pk in feeds:
feeds[moved_feed_id] = feeds[duplicate_feeds[0].feed_id]
feeds[moved_feed_id]['dupe_feed_id'] = duplicate_feeds[0].feed_id
if check_fetch_status:
missing_feed_ids = list(set(feed_ids) - set(sub_feed_ids))
if missing_feed_ids:
duplicate_feeds = DuplicateFeed.objects.filter(duplicate_feed_id__in=missing_feed_ids)
for duplicate_feed in duplicate_feeds:
feeds[duplicate_feed.duplicate_feed_id] = {'id': duplicate_feed.feed_id}
interactions_count = MInteraction.user_unread_count(user.pk)
if True or settings.DEBUG or check_fetch_status:
end = datetime.datetime.now()
extra_fetch = ""
if check_fetch_status or favicons_fetching:
extra_fetch = "(%s/%s)" % (check_fetch_status, len(favicons_fetching))
logging.user(request, "~FBRefreshing %s+%s feeds %s (%.4s/%.4s/%.4s)" % (
len(list(feeds.keys())), len(list(social_feeds.keys())), extra_fetch,
(checkpoint1-start).total_seconds(),
(checkpoint2-start).total_seconds(),
(end-start).total_seconds(),
))
MAnalyticsLoader.add(page_load=time.time()-start_time)
return {
'feeds': feeds,
'social_feeds': social_feeds,
'interactions_count': interactions_count,
}
@json.json_view
def interactions_count(request):
user = get_user(request)
interactions_count = MInteraction.user_unread_count(user.pk)
return {
'interactions_count': interactions_count,
}
@never_cache
@ajax_login_required
@json.json_view
def feed_unread_count(request):
get_post = getattr(request, request.method)
start = time.time()
user = request.user
feed_ids = get_post.getlist('feed_id') or get_post.getlist('feed_id[]')
force = request.GET.get('force', False)
social_feed_ids = [feed_id for feed_id in feed_ids if 'social:' in feed_id]
feed_ids = list(set(feed_ids) - set(social_feed_ids))
feeds = {}
if feed_ids:
feeds = UserSubscription.feeds_with_updated_counts(user, feed_ids=feed_ids, force=force)
social_feeds = {}
if social_feed_ids:
social_feeds = MSocialSubscription.feeds_with_updated_counts(user, social_feed_ids=social_feed_ids)
if len(feed_ids) == 1:
if settings.DEBUG:
feed_title = Feed.get_by_id(feed_ids[0]).feed_title
else:
feed_title = feed_ids[0]
elif len(social_feed_ids) == 1:
social_profile = MSocialProfile.objects.get(user_id=social_feed_ids[0].replace('social:', ''))
feed_title = social_profile.user.username if social_profile.user else "[deleted]"
else:
feed_title = "%s feeds" % (len(feeds) + len(social_feeds))
logging.user(request, "~FBUpdating unread count on: %s" % feed_title)
MAnalyticsLoader.add(page_load=time.time()-start)
return {'feeds': feeds, 'social_feeds': social_feeds}
def refresh_feed(request, feed_id):
start = time.time()
user = get_user(request)
feed = get_object_or_404(Feed, pk=feed_id)
feed = feed.update(force=True, compute_scores=False)
usersub = UserSubscription.objects.get(user=user, feed=feed)
usersub.calculate_feed_scores(silent=False)
logging.user(request, "~FBRefreshing feed: %s" % feed)
MAnalyticsLoader.add(page_load=time.time()-start)
return load_single_feed(request, feed_id)
@never_cache
@json.json_view
def load_single_feed(request, feed_id):
start = time.time()
user = get_user(request)
# offset = int(request.GET.get('offset', 0))
# limit = int(request.GET.get('limit', 6))
limit = 6
page = int(request.GET.get('page', 1))
delay = int(request.GET.get('delay', 0))
offset = limit * (page-1)
order = request.GET.get('order', 'newest')
read_filter = request.GET.get('read_filter', 'all')
query = request.GET.get('query', '').strip()
include_story_content = is_true(request.GET.get('include_story_content', True))
include_hidden = is_true(request.GET.get('include_hidden', False))
include_feeds = is_true(request.GET.get('include_feeds', False))
message = None
user_search = None
dupe_feed_id = None
user_profiles = []
now = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone)
if not feed_id: raise Http404
feed_address = request.GET.get('feed_address')
feed = Feed.get_by_id(feed_id, feed_address=feed_address)
if not feed:
raise Http404
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
usersub = None
if feed.is_newsletter and not usersub:
# User must be subscribed to a newsletter in order to read it
raise Http404
if feed.num_subscribers == 1 and not usersub and not user.is_staff:
# This feed could be private so user must be subscribed in order to read it
raise Http404
if page > 400:
logging.user(request, "~BR~FK~SBOver page 400 on single feed: %s" % page)
raise Http404
if query:
if user.profile.is_premium:
user_search = MUserSearch.get_user(user.pk)
user_search.touch_search_date()
stories = feed.find_stories(query, order=order, offset=offset, limit=limit)
else:
stories = []
message = "You must be a premium subscriber to search."
elif read_filter == 'starred':
mstories = MStarredStory.objects(
user_id=user.pk,
story_feed_id=feed_id
).order_by('%sstarred_date' % ('-' if order == 'newest' else ''))[offset:offset+limit]
stories = Feed.format_stories(mstories)
elif usersub and read_filter == 'unread':
stories = usersub.get_stories(order=order, read_filter=read_filter, offset=offset, limit=limit)
else:
stories = feed.get_stories(offset, limit, order=order)
checkpoint1 = time.time()
try:
stories, user_profiles = MSharedStory.stories_with_comments_and_profiles(stories, user.pk)
except redis.ConnectionError:
logging.user(request, "~BR~FK~SBRedis is unavailable for shared stories.")
checkpoint2 = time.time()
# Get intelligence classifier for user
if usersub and usersub.is_trained:
classifier_feeds = list(MClassifierFeed.objects(user_id=user.pk, feed_id=feed_id, social_user_id=0))
classifier_authors = list(MClassifierAuthor.objects(user_id=user.pk, feed_id=feed_id))
classifier_titles = list(MClassifierTitle.objects(user_id=user.pk, feed_id=feed_id))
classifier_tags = list(MClassifierTag.objects(user_id=user.pk, feed_id=feed_id))
else:
classifier_feeds = []
classifier_authors = []
classifier_titles = []
classifier_tags = []
classifiers = get_classifiers_for_user(user, feed_id=feed_id,
classifier_feeds=classifier_feeds,
classifier_authors=classifier_authors,
classifier_titles=classifier_titles,
classifier_tags=classifier_tags)
checkpoint3 = time.time()
unread_story_hashes = []
if stories:
if (read_filter == 'all' or query) and usersub:
unread_story_hashes = UserSubscription.story_hashes(user.pk, read_filter='unread',
feed_ids=[usersub.feed_id],
usersubs=[usersub],
cutoff_date=user.profile.unread_cutoff)
story_hashes = [story['story_hash'] for story in stories if story['story_hash']]
starred_stories = MStarredStory.objects(user_id=user.pk,
story_feed_id=feed.pk,
story_hash__in=story_hashes)\
.hint([('user_id', 1), ('story_hash', 1)])
shared_story_hashes = MSharedStory.check_shared_story_hashes(user.pk, story_hashes)
shared_stories = []
if shared_story_hashes:
shared_stories = MSharedStory.objects(user_id=user.pk,
story_hash__in=shared_story_hashes)\
.hint([('story_hash', 1)])\
.only('story_hash', 'shared_date', 'comments')
starred_stories = dict([(story.story_hash, story)
for story in starred_stories])
shared_stories = dict([(story.story_hash, dict(shared_date=story.shared_date,
comments=story.comments))
for story in shared_stories])
checkpoint4 = time.time()
for story in stories:
if not include_story_content:
del story['story_content']
story_date = localtime_for_timezone(story['story_date'], user.profile.timezone)
nowtz = localtime_for_timezone(now, user.profile.timezone)
story['short_parsed_date'] = format_story_link_date__short(story_date, nowtz)
story['long_parsed_date'] = format_story_link_date__long(story_date, nowtz)
if usersub:
story['read_status'] = 1
if not user.profile.is_archive and story['story_date'] < user.profile.unread_cutoff:
story['read_status'] = 1
elif (read_filter == 'all' or query) and usersub:
story['read_status'] = 1 if story['story_hash'] not in unread_story_hashes else 0
elif read_filter == 'unread' and usersub:
story['read_status'] = 0
if story['story_hash'] in starred_stories:
story['starred'] = True
starred_story = Feed.format_story(starred_stories[story['story_hash']])
starred_date = localtime_for_timezone(starred_story['starred_date'],
user.profile.timezone)
story['starred_date'] = format_story_link_date__long(starred_date, now)
story['starred_timestamp'] = int(starred_date.timestamp())
story['user_tags'] = starred_story['user_tags']
story['user_notes'] = starred_story['user_notes']
story['highlights'] = starred_story['highlights']
if story['story_hash'] in shared_stories:
story['shared'] = True
shared_date = localtime_for_timezone(shared_stories[story['story_hash']]['shared_date'],
user.profile.timezone)
story['shared_date'] = format_story_link_date__long(shared_date, now)
story['shared_comments'] = strip_tags(shared_stories[story['story_hash']]['comments'])
else:
story['read_status'] = 1
story['intelligence'] = {
'feed': apply_classifier_feeds(classifier_feeds, feed),
'author': apply_classifier_authors(classifier_authors, story),
'tags': apply_classifier_tags(classifier_tags, story),
'title': apply_classifier_titles(classifier_titles, story),
}
story['score'] = UserSubscription.score_story(story['intelligence'])
# Intelligence
feed_tags = json.decode(feed.data.popular_tags) if feed.data.popular_tags else []
feed_authors = json.decode(feed.data.popular_authors) if feed.data.popular_authors else []
if include_feeds:
feeds = Feed.objects.filter(pk__in=set([story['story_feed_id'] for story in stories]))
feeds = [f.canonical(include_favicon=False) for f in feeds]
if usersub:
usersub.feed_opens += 1
usersub.needs_unread_recalc = True
try:
usersub.save(update_fields=['feed_opens', 'needs_unread_recalc'])
except DatabaseError as e:
logging.user(request, f"~BR~FK~SBNo changes in usersub, ignoring... {e}")
diff1 = checkpoint1-start
diff2 = checkpoint2-start
diff3 = checkpoint3-start
diff4 = checkpoint4-start
timediff = time.time()-start
last_update = relative_timesince(feed.last_update)
time_breakdown = ""
if timediff > 1 or settings.DEBUG:
time_breakdown = "~SN~FR(~SB%.4s/%.4s/%.4s/%.4s~SN)" % (
diff1, diff2, diff3, diff4)
search_log = "~SN~FG(~SB%s~SN) " % query if query else ""
logging.user(request, "~FYLoading feed: ~SB%s%s (%s/%s) %s%s" % (
feed.feed_title[:22], ('~SN/p%s' % page) if page > 1 else '', order, read_filter, search_log, time_breakdown))
MAnalyticsLoader.add(page_load=timediff)
if hasattr(request, 'start_time'):
seconds = time.time() - request.start_time
RStats.add('page_load', duration=seconds)
if not include_hidden:
hidden_stories_removed = 0
new_stories = []
for story in stories:
if story['score'] >= 0:
new_stories.append(story)
else:
hidden_stories_removed += 1
stories = new_stories
data = dict(stories=stories,
user_profiles=user_profiles,
feed_tags=feed_tags,
feed_authors=feed_authors,
classifiers=classifiers,
updated=last_update,
user_search=user_search,
feed_id=feed.pk,
elapsed_time=round(float(timediff), 2),
message=message)
if include_feeds: data['feeds'] = feeds
if not include_hidden: data['hidden_stories_removed'] = hidden_stories_removed
if dupe_feed_id: data['dupe_feed_id'] = dupe_feed_id
if not usersub:
data.update(feed.canonical())
# if not usersub and feed.num_subscribers <= 1:
# data = dict(code=-1, message="You must be subscribed to this feed.")
# time.sleep(random.randint(1, 3))
if delay and user.is_staff:
# time.sleep(random.randint(2, 7) / 10.0)
time.sleep(delay)
# if page == 1:
# time.sleep(1)
# else:
# time.sleep(20)
# if page == 2:
# assert False
return data
def load_feed_page(request, feed_id):
if not feed_id:
raise Http404
feed = Feed.get_by_id(feed_id)
if feed and feed.has_page and not feed.has_page_exception:
if settings.BACKED_BY_AWS.get('pages_on_node'):
domain = Site.objects.get_current().domain
url = "https://%s/original_page/%s" % (
domain,
feed.pk,
)
try:
page_response = requests.get(url, verify=not settings.DEBUG)
except requests.ConnectionError as e:
logging.user(request, f"~FR~SBError loading original page: {url} {e}")
page_response = None
if page_response and page_response.status_code == 200:
response = HttpResponse(page_response.content, content_type="text/html; charset=utf-8")
response['Content-Encoding'] = 'deflate'
response['Last-Modified'] = page_response.headers.get('Last-modified')
response['Etag'] = page_response.headers.get('Etag')
response['Content-Length'] = str(len(page_response.content))
logging.user(request, "~FYLoading original page (%s), proxied from node: ~SB%s bytes" %
(feed_id, len(page_response.content)))
return response
if settings.BACKED_BY_AWS['pages_on_s3'] and feed.s3_page:
if settings.PROXY_S3_PAGES:
key = settings.S3_CONN.Bucket(settings.S3_PAGES_BUCKET_NAME).Object(key=feed.s3_pages_key)
if key:
compressed_data = key.get()["Body"]
response = HttpResponse(compressed_data, content_type="text/html; charset=utf-8")
response['Content-Encoding'] = 'gzip'
logging.user(request, "~FYLoading original page, proxied: ~SB%s bytes" %
(len(compressed_data)))
return response
else:
logging.user(request, "~FYLoading original page, non-proxied")
return HttpResponseRedirect('//%s/%s' % (settings.S3_PAGES_BUCKET_NAME,
feed.s3_pages_key))
data = MFeedPage.get_data(feed_id=feed_id)
if not data or not feed or not feed.has_page or feed.has_page_exception:
logging.user(request, "~FYLoading original page, ~FRmissing")
return render(request, 'static/404_original_page.xhtml', {},
content_type='text/html',
status=404)
logging.user(request, "~FYLoading original page, from the db")
return HttpResponse(data, content_type="text/html; charset=utf-8")
@json.json_view
def load_starred_stories(request):
user = get_user(request)
offset = int(request.GET.get('offset', 0))
limit = int(request.GET.get('limit', 10))
page = int(request.GET.get('page', 0))
query = request.GET.get('query', '').strip()
order = request.GET.get('order', 'newest')
tag = request.GET.get('tag')
highlights = is_true(request.GET.get('highlights', False))
story_hashes = request.GET.getlist('h') or request.GET.getlist('h[]')
story_hashes = story_hashes[:100]
version = int(request.GET.get('v', 1))
now = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone)
message = None
order_by = '-' if order == "newest" else ""
if page: offset = limit * (page - 1)
if query:
# results = SearchStarredStory.query(user.pk, query)
# story_ids = [result.db_id for result in results]
if user.profile.is_premium:
stories = MStarredStory.find_stories(query, user.pk, tag=tag, offset=offset, limit=limit,
order=order)
else:
stories = []
message = "You must be a premium subscriber to search."
elif highlights:
if user.profile.is_premium:
mstories = MStarredStory.objects(
user_id=user.pk,
highlights__exists=True,
__raw__={"$where": "this.highlights.length > 0"}
).order_by('%sstarred_date' % order_by)[offset:offset+limit]
stories = Feed.format_stories(mstories)
else:
stories = []
message = "You must be a premium subscriber to read through saved story highlights."
elif tag:
if user.profile.is_premium:
mstories = MStarredStory.objects(
user_id=user.pk,
user_tags__contains=tag
).order_by('%sstarred_date' % order_by)[offset:offset+limit]
stories = Feed.format_stories(mstories)
else:
stories = []
message = "You must be a premium subscriber to read saved stories by tag."
elif story_hashes:
limit = 100
mstories = MStarredStory.objects(
user_id=user.pk,
story_hash__in=story_hashes
).order_by('%sstarred_date' % order_by)[offset:offset+limit]
stories = Feed.format_stories(mstories)
else:
mstories = MStarredStory.objects(
user_id=user.pk
).order_by('%sstarred_date' % order_by)[offset:offset+limit]
stories = Feed.format_stories(mstories)
stories, user_profiles = MSharedStory.stories_with_comments_and_profiles(stories, user.pk, check_all=True)
story_hashes = [story['story_hash'] for story in stories]
story_feed_ids = list(set(s['story_feed_id'] for s in stories))
usersub_ids = UserSubscription.objects.filter(user__pk=user.pk, feed__pk__in=story_feed_ids).values('feed__pk')
usersub_ids = [us['feed__pk'] for us in usersub_ids]
unsub_feed_ids = list(set(story_feed_ids).difference(set(usersub_ids)))
unsub_feeds = Feed.objects.filter(pk__in=unsub_feed_ids)
unsub_feeds = dict((feed.pk, feed.canonical(include_favicon=False)) for feed in unsub_feeds)
for story in stories:
if story['story_feed_id'] in unsub_feeds: continue
duplicate_feed = DuplicateFeed.objects.filter(duplicate_feed_id=story['story_feed_id'])
if not duplicate_feed: continue
feed_id = duplicate_feed[0].feed_id
try:
saved_story = MStarredStory.objects.get(user_id=user.pk, story_hash=story['story_hash'])
saved_story.feed_id = feed_id
_, story_hash = MStory.split_story_hash(story['story_hash'])
saved_story.story_hash = "%s:%s" % (feed_id, story_hash)
saved_story.story_feed_id = feed_id
story['story_hash'] = saved_story.story_hash
story['story_feed_id'] = saved_story.story_feed_id
saved_story.save()
logging.user(request, "~FCSaving new feed for starred story: ~SB%s -> %s" % (story['story_hash'], feed_id))
except (MStarredStory.DoesNotExist, MStarredStory.MultipleObjectsReturned):
logging.user(request, "~FCCan't find feed for starred story: ~SB%s" % (story['story_hash']))
continue
shared_story_hashes = MSharedStory.check_shared_story_hashes(user.pk, story_hashes)
shared_stories = []
if shared_story_hashes:
shared_stories = MSharedStory.objects(user_id=user.pk,
story_hash__in=shared_story_hashes)\
.hint([('story_hash', 1)])\
.only('story_hash', 'shared_date', 'comments')
shared_stories = dict([(story.story_hash, dict(shared_date=story.shared_date,
comments=story.comments))
for story in shared_stories])
nowtz = localtime_for_timezone(now, user.profile.timezone)
for story in stories:
story_date = localtime_for_timezone(story['story_date'], user.profile.timezone)
story['short_parsed_date'] = format_story_link_date__short(story_date, nowtz)
story['long_parsed_date'] = format_story_link_date__long(story_date, nowtz)
starred_date = localtime_for_timezone(story['starred_date'], user.profile.timezone)
story['starred_date'] = format_story_link_date__long(starred_date, nowtz)
story['starred_timestamp'] = int(starred_date.timestamp())
story['read_status'] = 1
story['starred'] = True
story['intelligence'] = {
'feed': 1,
'author': 0,
'tags': 0,
'title': 0,
}
if story['story_hash'] in shared_stories:
story['shared'] = True
story['shared_comments'] = strip_tags(shared_stories[story['story_hash']]['comments'])
search_log = "~SN~FG(~SB%s~SN)" % query if query else ""
logging.user(request, "~FCLoading starred stories: ~SB%s stories %s" % (len(stories), search_log))
return {
"stories": stories,
"user_profiles": user_profiles,
'feeds': list(unsub_feeds.values()) if version == 2 else unsub_feeds,
"message": message,
}
@json.json_view
def starred_story_hashes(request):
user = get_user(request)
include_timestamps = is_true(request.GET.get('include_timestamps', False))
mstories = MStarredStory.objects(
user_id=user.pk
).only('story_hash', 'starred_date', 'starred_updated').order_by('-starred_date')
if include_timestamps:
story_hashes = []
for s in mstories:
date = s.starred_date
if s.starred_updated:
date = s.starred_updated
story_hashes.append((s.story_hash, date.strftime("%s")))
else:
story_hashes = [s.story_hash for s in mstories]
logging.user(request, "~FYLoading ~FCstarred story hashes~FY: %s story hashes" %
(len(story_hashes)))
return dict(starred_story_hashes=story_hashes)
def starred_stories_rss_feed(request, user_id, secret_token):
return starred_stories_rss_feed_tag(request, user_id, secret_token, tag_slug=None)
def starred_stories_rss_feed_tag(request, user_id, secret_token, tag_slug):
try:
user = User.objects.get(pk=user_id)
except User.DoesNotExist:
raise Http404
if tag_slug:
try:
tag_counts = MStarredStoryCounts.objects.get(user_id=user_id, slug=tag_slug)
except MStarredStoryCounts.MultipleObjectsReturned:
tag_counts = MStarredStoryCounts.objects(user_id=user_id, slug=tag_slug).first()
except MStarredStoryCounts.DoesNotExist:
raise Http404
else:
_, starred_count = MStarredStoryCounts.user_counts(user.pk, include_total=True)
data = {}
if tag_slug:
data['title'] = "Saved Stories - %s" % tag_counts.tag
else:
data['title'] = "Saved Stories"
data['link'] = "%s%s" % (
settings.NEWSBLUR_URL,
reverse('saved-stories-tag', kwargs=dict(tag_name=tag_slug)))
if tag_slug:
data['description'] = "Stories saved by %s on NewsBlur with the tag \"%s\"." % (user.username,
tag_counts.tag)
else:
data['description'] = "Stories saved by %s on NewsBlur." % (user.username)
data['lastBuildDate'] = datetime.datetime.utcnow()
data['generator'] = 'NewsBlur - %s' % settings.NEWSBLUR_URL
data['docs'] = None
data['author_name'] = user.username
data['feed_url'] = "%s%s" % (
settings.NEWSBLUR_URL,
reverse('starred-stories-rss-feed-tag',
kwargs=dict(user_id=user_id, secret_token=secret_token, tag_slug=tag_slug)),
)
rss = feedgenerator.Atom1Feed(**data)
if not tag_slug or not tag_counts.tag:
starred_stories = MStarredStory.objects(
user_id=user.pk
).order_by('-starred_date').limit(25)
elif tag_counts.is_highlights:
starred_stories = MStarredStory.objects(
user_id=user.pk,
highlights__exists=True,
__raw__={"$where": "this.highlights.length > 0"}
).order_by('-starred_date').limit(25)
else:
starred_stories = MStarredStory.objects(
user_id=user.pk,
user_tags__contains=tag_counts.tag
).order_by('-starred_date').limit(25)
starred_stories = Feed.format_stories(starred_stories)
for starred_story in starred_stories:
story_data = {
'title': smart_str(starred_story['story_title']),
'link': starred_story['story_permalink'],
'description': smart_str(starred_story['story_content']),
'author_name': starred_story['story_authors'],
'categories': starred_story['story_tags'],
'unique_id': starred_story['story_permalink'],
'pubdate': starred_story['starred_date'],
}
rss.add_item(**story_data)
logging.user(request, "~FBGenerating ~SB%s~SN's saved story RSS feed (%s, %s stories): ~FM%s" % (
user.username,
tag_counts.tag if tag_slug else "[All stories]",
tag_counts.count if tag_slug else starred_count,
request.META.get('HTTP_USER_AGENT', "")[:24]
))
return HttpResponse(rss.writeString('utf-8'), content_type='application/rss+xml')
def folder_rss_feed(request, user_id, secret_token, unread_filter, folder_slug):
domain = Site.objects.get_current().domain
date_hack_2023 = (datetime.datetime.now() > datetime.datetime(2023, 7, 1))
try:
user = User.objects.get(pk=user_id)
except User.DoesNotExist:
raise Http404
user_sub_folders = get_object_or_404(UserSubscriptionFolders, user=user)
feed_ids, folder_title = user_sub_folders.feed_ids_under_folder_slug(folder_slug)
usersubs = UserSubscription.subs_for_feeds(user.pk, feed_ids=feed_ids)
if feed_ids and ((user.profile.is_archive and date_hack_2023) or (not date_hack_2023)):
params = {
"user_id": user.pk,
"feed_ids": feed_ids,
"offset": 0,
"limit": 20,
"order": 'newest',
"read_filter": 'all',
"cache_prefix": "RSS:"
}
story_hashes, unread_feed_story_hashes = UserSubscription.feed_stories(**params)
else:
story_hashes = []
mstories = MStory.objects(story_hash__in=story_hashes).order_by('-story_date')
stories = Feed.format_stories(mstories)
filtered_stories = []
found_feed_ids = list(set([story['story_feed_id'] for story in stories]))
trained_feed_ids = [sub.feed_id for sub in usersubs if sub.is_trained]
found_trained_feed_ids = list(set(trained_feed_ids) & set(found_feed_ids))
if found_trained_feed_ids:
classifier_feeds = list(MClassifierFeed.objects(user_id=user.pk,
feed_id__in=found_trained_feed_ids,
social_user_id=0))
classifier_authors = list(MClassifierAuthor.objects(user_id=user.pk,
feed_id__in=found_trained_feed_ids))
classifier_titles = list(MClassifierTitle.objects(user_id=user.pk,
feed_id__in=found_trained_feed_ids))
classifier_tags = list(MClassifierTag.objects(user_id=user.pk,
feed_id__in=found_trained_feed_ids))
else:
classifier_feeds = []
classifier_authors = []
classifier_titles = []
classifier_tags = []
sort_classifiers_by_feed(user=user, feed_ids=found_feed_ids,
classifier_feeds=classifier_feeds,
classifier_authors=classifier_authors,
classifier_titles=classifier_titles,
classifier_tags=classifier_tags)
for story in stories:
story['intelligence'] = {
'feed': apply_classifier_feeds(classifier_feeds, story['story_feed_id']),
'author': apply_classifier_authors(classifier_authors, story),
'tags': apply_classifier_tags(classifier_tags, story),
'title': apply_classifier_titles(classifier_titles, story),
}
story['score'] = UserSubscription.score_story(story['intelligence'])
if unread_filter == 'focus' and story['score'] >= 1:
filtered_stories.append(story)
elif unread_filter == 'unread' and story['score'] >= 0:
filtered_stories.append(story)
stories = filtered_stories
data = {}
data['title'] = "%s from %s (%s sites)" % (folder_title, user.username, len(feed_ids))
data['link'] = "https://%s%s" % (
domain,
reverse('folder', kwargs=dict(folder_name=folder_title)))
data['description'] = "Unread stories in %s on NewsBlur. From %s's account and contains %s sites." % (
folder_title,
user.username,
len(feed_ids))
data['lastBuildDate'] = datetime.datetime.utcnow()
data['generator'] = 'NewsBlur - %s' % settings.NEWSBLUR_URL
data['docs'] = None
data['author_name'] = user.username
data['feed_url'] = "https://%s%s" % (
domain,
reverse('folder-rss-feed',
kwargs=dict(user_id=user_id, secret_token=secret_token, unread_filter=unread_filter, folder_slug=folder_slug)),
)
rss = feedgenerator.Atom1Feed(**data)
for story in stories:
feed = Feed.get_by_id(story['story_feed_id'])
feed_title = feed.feed_title if feed else ""
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
if usersub.user_title:
feed_title = usersub.user_title
except UserSubscription.DoesNotExist:
usersub = None
story_content = """%s
%s""" % (
smart_str(story['story_content']),
Site.objects.get_current().domain,
story['story_feed_id'],
feed_title,
)
story_content = re.sub(r'[\x00-\x08\x0B-\x0C\x0E-\x1F]', '', story_content)
story_title = "%s%s" % (("%s: " % feed_title) if feed_title else "", story['story_title'])
story_data = {
'title': story_title,
'link': story['story_permalink'],
'description': story_content,
'categories': story['story_tags'],
'unique_id': 'https://%s/site/%s/%s/' % (domain, story['story_feed_id'], story['guid_hash']),
'pubdate': localtime_for_timezone(story['story_date'], user.profile.timezone),
}
if story['story_authors']:
story_data['author_name'] = story['story_authors']
rss.add_item(**story_data)
# TODO: Remove below date hack to accomodate users who paid for premium but want folder rss
if not user.profile.is_archive and date_hack_2023:
story_data = {
'title': "You must have a premium archive subscription on NewsBlur to have RSS feeds for folders.",
'link': "https://%s/?next=premium" % domain,
'description': "You must have a premium archive subscription on NewsBlur to have RSS feeds for folders.",
'unique_id': "https://%s/premium_only" % domain,
'pubdate': localtime_for_timezone(datetime.datetime.now(), user.profile.timezone),
}
rss.add_item(**story_data)
logging.user(request, "~FBGenerating ~SB%s~SN's folder RSS feed (%s, %s stories): ~FM%s" % (
user.username,
folder_title,
len(stories),
request.META.get('HTTP_USER_AGENT', "")[:24]
))
return HttpResponse(rss.writeString('utf-8'), content_type='application/rss+xml')
@json.json_view
def load_read_stories(request):
user = get_user(request)
offset = int(request.GET.get('offset', 0))
limit = int(request.GET.get('limit', 10))
page = int(request.GET.get('page', 0))
order = request.GET.get('order', 'newest')
query = request.GET.get('query', '').strip()
now = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone)
message = None
if page: offset = limit * (page - 1)
if query:
stories = []
message = "Not implemented yet."
# if user.profile.is_premium:
# stories = MStarredStory.find_stories(query, user.pk, offset=offset, limit=limit)
# else:
# stories = []
# message = "You must be a premium subscriber to search."
else:
story_hashes = RUserStory.get_read_stories(user.pk, offset=offset, limit=limit, order=order)
mstories = MStory.objects(story_hash__in=story_hashes)
stories = Feed.format_stories(mstories)
stories = sorted(stories, key=lambda story: story_hashes.index(story['story_hash']),
reverse=bool(order=="oldest"))
stories, user_profiles = MSharedStory.stories_with_comments_and_profiles(stories, user.pk, check_all=True)
story_hashes = [story['story_hash'] for story in stories]
story_feed_ids = list(set(s['story_feed_id'] for s in stories))
usersub_ids = UserSubscription.objects.filter(user__pk=user.pk, feed__pk__in=story_feed_ids).values('feed__pk')
usersub_ids = [us['feed__pk'] for us in usersub_ids]
unsub_feed_ids = list(set(story_feed_ids).difference(set(usersub_ids)))
unsub_feeds = Feed.objects.filter(pk__in=unsub_feed_ids)
unsub_feeds = [feed.canonical(include_favicon=False) for feed in unsub_feeds]
shared_stories = MSharedStory.objects(user_id=user.pk,
story_hash__in=story_hashes)\
.hint([('story_hash', 1)])\
.only('story_hash', 'shared_date', 'comments')
shared_stories = dict([(story.story_hash, dict(shared_date=story.shared_date,
comments=story.comments))
for story in shared_stories])
starred_stories = MStarredStory.objects(user_id=user.pk,
story_hash__in=story_hashes)\
.hint([('user_id', 1), ('story_hash', 1)])
starred_stories = dict([(story.story_hash, story)
for story in starred_stories])
nowtz = localtime_for_timezone(now, user.profile.timezone)
for story in stories:
story_date = localtime_for_timezone(story['story_date'], user.profile.timezone)
story['short_parsed_date'] = format_story_link_date__short(story_date, nowtz)
story['long_parsed_date'] = format_story_link_date__long(story_date, nowtz)
story['read_status'] = 1
story['intelligence'] = {
'feed': 1,
'author': 0,
'tags': 0,
'title': 0,
}
if story['story_hash'] in starred_stories:
story['starred'] = True
starred_story = Feed.format_story(starred_stories[story['story_hash']])
starred_date = localtime_for_timezone(starred_story['starred_date'],
user.profile.timezone)
story['starred_date'] = format_story_link_date__long(starred_date, now)
story['starred_timestamp'] = int(starred_date.timestamp())
if story['story_hash'] in shared_stories:
story['shared'] = True
story['shared_comments'] = strip_tags(shared_stories[story['story_hash']]['comments'])
search_log = "~SN~FG(~SB%s~SN)" % query if query else ""
logging.user(request, "~FCLoading read stories: ~SB%s stories %s" % (len(stories), search_log))
return {
"stories": stories,
"user_profiles": user_profiles,
"feeds": unsub_feeds,
"message": message,
}
@json.json_view
def load_river_stories__redis(request):
# get_post is request.REQUEST, since this endpoint needs to handle either
# GET or POST requests, since the parameters for this endpoint can be
# very long, at which point the max size of a GET url request is exceeded.
get_post = getattr(request, request.method)
limit = int(get_post.get('limit', 12))
start = time.time()
user = get_user(request)
message = None
feed_ids = get_post.getlist('feeds') or get_post.getlist('feeds[]')
feed_ids = [int(feed_id) for feed_id in feed_ids if feed_id]
if not feed_ids:
feed_ids = get_post.getlist('f') or get_post.getlist('f[]')
feed_ids = [int(feed_id) for feed_id in get_post.getlist('f') if feed_id]
story_hashes = get_post.getlist('h') or get_post.getlist('h[]')
story_hashes = story_hashes[:100]
requested_hashes = len(story_hashes)
original_feed_ids = list(feed_ids)
page = int(get_post.get('page', 1))
order = get_post.get('order', 'newest')
read_filter = get_post.get('read_filter', 'unread')
query = get_post.get('query', '').strip()
include_hidden = is_true(get_post.get('include_hidden', False))
include_feeds = is_true(get_post.get('include_feeds', False))
on_dashboard = is_true(get_post.get('dashboard', False)) or is_true(get_post.get('on_dashboard', False))
infrequent = is_true(get_post.get('infrequent', False))
if infrequent:
infrequent = get_post.get('infrequent')
now = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone)
usersubs = []
code = 1
user_search = None
offset = (page-1) * limit
story_date_order = "%sstory_date" % ('' if order == 'oldest' else '-')
if user.pk == 86178:
# Disable Michael_Novakhov account
logging.user(request, "~FCLoading ~FMMichael_Novakhov~SN's river, resource usage too high, ignoring.")
return HttpResponse("Resource usage too high", status=429)
if infrequent:
feed_ids = Feed.low_volume_feeds(feed_ids, stories_per_month=infrequent)
if story_hashes:
unread_feed_story_hashes = None
read_filter = 'all'
mstories = MStory.objects(story_hash__in=story_hashes).order_by(story_date_order)
stories = Feed.format_stories(mstories)
elif query:
if user.profile.is_premium:
user_search = MUserSearch.get_user(user.pk)
user_search.touch_search_date()
usersubs = UserSubscription.subs_for_feeds(user.pk, feed_ids=feed_ids,
read_filter='all')
feed_ids = [sub.feed_id for sub in usersubs]
if infrequent:
feed_ids = Feed.low_volume_feeds(feed_ids, stories_per_month=infrequent)
stories = Feed.find_feed_stories(feed_ids, query, order=order, offset=offset, limit=limit)
mstories = stories
unread_feed_story_hashes = UserSubscription.story_hashes(user.pk, feed_ids=feed_ids,
read_filter="unread", order=order,
cutoff_date=user.profile.unread_cutoff)
else:
stories = []
mstories = []
message = "You must be a premium subscriber to search."
elif read_filter == 'starred':
mstories = MStarredStory.objects(
user_id=user.pk,
story_feed_id__in=feed_ids
).order_by('%sstarred_date' % ('-' if order == 'newest' else ''))[offset:offset+limit]
stories = Feed.format_stories(mstories)
else:
usersubs = UserSubscription.subs_for_feeds(user.pk, feed_ids=feed_ids,
read_filter=read_filter)
all_feed_ids = [f for f in feed_ids]
feed_ids = [sub.feed_id for sub in usersubs]
if infrequent:
feed_ids = Feed.low_volume_feeds(feed_ids, stories_per_month=infrequent)
if feed_ids:
params = {
"user_id": user.pk,
"feed_ids": feed_ids,
"all_feed_ids": all_feed_ids,
"offset": offset,
"limit": limit,
"order": order,
"read_filter": read_filter,
"usersubs": usersubs,
"cutoff_date": user.profile.unread_cutoff,
"cache_prefix": "dashboard:" if on_dashboard else "",
}
story_hashes, unread_feed_story_hashes = UserSubscription.feed_stories(**params)
else:
story_hashes = []
unread_feed_story_hashes = []
mstories = MStory.objects(story_hash__in=story_hashes[:limit]).order_by(story_date_order)
stories = Feed.format_stories(mstories)
found_feed_ids = list(set([story['story_feed_id'] for story in stories]))
stories, user_profiles = MSharedStory.stories_with_comments_and_profiles(stories, user.pk)
if not usersubs:
usersubs = UserSubscription.subs_for_feeds(user.pk, feed_ids=found_feed_ids,
read_filter=read_filter)
trained_feed_ids = [sub.feed_id for sub in usersubs if sub.is_trained]
found_trained_feed_ids = list(set(trained_feed_ids) & set(found_feed_ids))
# Find starred stories
if found_feed_ids:
if read_filter == 'starred':
starred_stories = mstories
else:
story_hashes = [s['story_hash'] for s in stories]
starred_stories = MStarredStory.objects(
user_id=user.pk,
story_hash__in=story_hashes)
starred_stories = dict([(story.story_hash, dict(starred_date=story.starred_date,
user_tags=story.user_tags,
highlights=story.highlights,
user_notes=story.user_notes))
for story in starred_stories])
else:
starred_stories = {}
# Intelligence classifiers for all feeds involved
if found_trained_feed_ids:
classifier_feeds = list(MClassifierFeed.objects(user_id=user.pk,
feed_id__in=found_trained_feed_ids,
social_user_id=0))
classifier_authors = list(MClassifierAuthor.objects(user_id=user.pk,
feed_id__in=found_trained_feed_ids))
classifier_titles = list(MClassifierTitle.objects(user_id=user.pk,
feed_id__in=found_trained_feed_ids))
classifier_tags = list(MClassifierTag.objects(user_id=user.pk,
feed_id__in=found_trained_feed_ids))
else:
classifier_feeds = []
classifier_authors = []
classifier_titles = []
classifier_tags = []
classifiers = sort_classifiers_by_feed(user=user, feed_ids=found_feed_ids,
classifier_feeds=classifier_feeds,
classifier_authors=classifier_authors,
classifier_titles=classifier_titles,
classifier_tags=classifier_tags)
# Just need to format stories
nowtz = localtime_for_timezone(now, user.profile.timezone)
for story in stories:
if read_filter == 'starred':
story['read_status'] = 1
else:
story['read_status'] = 0
if read_filter == 'all' or query:
if (unread_feed_story_hashes is not None and
story['story_hash'] not in unread_feed_story_hashes):
story['read_status'] = 1
story_date = localtime_for_timezone(story['story_date'], user.profile.timezone)
story['short_parsed_date'] = format_story_link_date__short(story_date, nowtz)
story['long_parsed_date'] = format_story_link_date__long(story_date, nowtz)
if story['story_hash'] in starred_stories:
story['starred'] = True
starred_date = localtime_for_timezone(starred_stories[story['story_hash']]['starred_date'],
user.profile.timezone)
story['starred_date'] = format_story_link_date__long(starred_date, now)
story['starred_timestamp'] = int(starred_date.timestamp())
story['user_tags'] = starred_stories[story['story_hash']]['user_tags']
story['user_notes'] = starred_stories[story['story_hash']]['user_notes']
story['highlights'] = starred_stories[story['story_hash']]['highlights']
story['intelligence'] = {
'feed': apply_classifier_feeds(classifier_feeds, story['story_feed_id']),
'author': apply_classifier_authors(classifier_authors, story),
'tags': apply_classifier_tags(classifier_tags, story),
'title': apply_classifier_titles(classifier_titles, story),
}
story['score'] = UserSubscription.score_story(story['intelligence'])
if include_feeds:
feeds = Feed.objects.filter(pk__in=set([story['story_feed_id'] for story in stories]))
feeds = [feed.canonical(include_favicon=False) for feed in feeds]
if not user.profile.is_premium and not include_feeds:
message = "The full River of News is a premium feature."
code = 0
# if page > 1:
# stories = []
# else:
# stories = stories[:5]
if not include_hidden:
hidden_stories_removed = 0
new_stories = []
for story in stories:
if story['score'] >= 0:
new_stories.append(story)
else:
hidden_stories_removed += 1
stories = new_stories
# if page > 1:
# import random
# time.sleep(random.randint(10, 16))
diff = time.time() - start
timediff = round(float(diff), 2)
if requested_hashes and story_hashes:
logging.user(request, "~FB%sLoading ~FC%s~FB stories: %s%s" %
("~FBAuto-" if on_dashboard else "",
requested_hashes, story_hashes[:3], f"...(+{len(story_hashes)-3})" if len(story_hashes) > 3 else ""))
else:
logging.user(request, "~FY%sLoading ~FC%sriver stories~FY: ~SBp%s~SN (%s/%s "
"stories, ~SN%s/%s/%s feeds, %s/%s)" %
("~FCAuto-" if on_dashboard else "",
"~FB~SBinfrequent~SN~FC " if infrequent else "",
page, len(stories), len(mstories), len(found_feed_ids),
len(feed_ids), len(original_feed_ids), order, read_filter))
if not on_dashboard and not (requested_hashes and story_hashes):
MAnalyticsLoader.add(page_load=diff) # Only count full pages, not individual stories
if hasattr(request, 'start_time'):
seconds = time.time() - request.start_time
RStats.add('page_load', duration=seconds)
data = dict(code=code,
message=message,
stories=stories,
classifiers=classifiers,
elapsed_time=timediff,
user_search=user_search,
user_profiles=user_profiles)
if include_feeds: data['feeds'] = feeds
if not include_hidden: data['hidden_stories_removed'] = hidden_stories_removed
return data
@json.json_view
def load_river_stories_widget(request):
logging.user(request, "Widget load")
river_stories_data = json.decode(load_river_stories__redis(request).content)
timeout = 3
start = time.time()
def load_url(url):
original_url = url
url = urllib.parse.urljoin(settings.NEWSBLUR_URL, url)
scontext = ssl.SSLContext(ssl.PROTOCOL_TLS)
scontext.verify_mode = ssl.VerifyMode.CERT_NONE
conn = None
try:
conn = urllib.request.urlopen(url, context=scontext, timeout=timeout)
except (urllib.error.URLError, socket.timeout):
pass
if not conn:
# logging.user(request.user, '"%s" wasn\'t fetched, trying again: %s' % (url, e))
url = url.replace('localhost', 'haproxy')
try:
conn = urllib.request.urlopen(url, context=scontext, timeout=timeout)
except (urllib.error.HTTPError, urllib.error.URLError, socket.timeout) as e:
logging.user(request.user, '~FB"%s" ~FRnot fetched~FB in %ss: ~SB%s' % (url, (time.time() - start), e))
return None
data = conn.read()
if not url.startswith("data:"):
data = base64.b64encode(data).decode('utf-8')
logging.user(request.user, '~FB"%s" ~SBfetched~SN in ~SB%ss' % (url, (time.time() - start)))
return dict(url=original_url, data=data)
# Find the image thumbnails and download in parallel
thumbnail_urls = []
for story in river_stories_data['stories']:
thumbnail_values = list(story['secure_image_thumbnails'].values())
for thumbnail_value in thumbnail_values:
if 'data:' in thumbnail_value:
continue
thumbnail_urls.append(thumbnail_value)
break
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:
pages = executor.map(load_url, thumbnail_urls)
# Reassemble thumbnails back into stories
thumbnail_data = dict()
for page in pages:
if not page: continue
thumbnail_data[page['url']] = page['data']
for story in river_stories_data['stories']:
thumbnail_values = list(story['secure_image_thumbnails'].values())
if thumbnail_values and thumbnail_values[0] in thumbnail_data:
page_url = thumbnail_values[0]
story['select_thumbnail_data'] = thumbnail_data[page_url]
logging.user(request, ("Elapsed Time: %ss" % (time.time() - start)))
return river_stories_data
@json.json_view
def complete_river(request):
user = get_user(request)
feed_ids = request.POST.getlist('feeds') or request.POST.getlist('feeds[]')
feed_ids = [int(feed_id) for feed_id in feed_ids if feed_id and feed_id.isnumeric()]
page = int(request.POST.get('page', 1))
read_filter = request.POST.get('read_filter', 'unread')
stories_truncated = 0
usersubs = UserSubscription.subs_for_feeds(user.pk, feed_ids=feed_ids,
read_filter=read_filter)
feed_ids = [sub.feed_id for sub in usersubs]
if feed_ids:
stories_truncated = UserSubscription.truncate_river(user.pk, feed_ids, read_filter, cache_prefix="dashboard:")
if page >= 1:
logging.user(request, "~FC~BBRiver complete on page ~SB%s~SN, truncating ~SB%s~SN stories from ~SB%s~SN feeds" % (page, stories_truncated, len(feed_ids)))
return dict(code=1, message="Truncated %s stories from %s" % (stories_truncated, len(feed_ids)))
@json.json_view
def unread_story_hashes(request):
user = get_user(request)
feed_ids = request.GET.getlist('feed_id') or request.GET.getlist('feed_id[]')
feed_ids = [int(feed_id) for feed_id in feed_ids if feed_id]
include_timestamps = is_true(request.GET.get('include_timestamps', False))
order = request.GET.get('order', 'newest')
read_filter = request.GET.get('read_filter', 'unread')
story_hashes = UserSubscription.story_hashes(user.pk, feed_ids=feed_ids,
order=order, read_filter=read_filter,
include_timestamps=include_timestamps,
group_by_feed=True,
cutoff_date=user.profile.unread_cutoff)
logging.user(request, "~FYLoading ~FCunread story hashes~FY: ~SB%s feeds~SN (%s story hashes)" %
(len(feed_ids), len(story_hashes)))
return dict(unread_feed_story_hashes=story_hashes)
@ajax_login_required
@json.json_view
def mark_all_as_read(request):
code = 1
try:
days = int(request.POST.get('days', 0))
except ValueError:
return dict(code=-1, message="Days parameter must be an integer, not: %s" %
request.POST.get('days'))
read_date = datetime.datetime.utcnow() - datetime.timedelta(days=days)
feeds = UserSubscription.objects.filter(user=request.user)
infrequent = is_true(request.POST.get('infrequent', False))
if infrequent:
infrequent = request.POST.get('infrequent')
feed_ids = Feed.low_volume_feeds([usersub.feed.pk for usersub in feeds], stories_per_month=infrequent)
feeds = UserSubscription.objects.filter(user=request.user, feed_id__in=feed_ids)
socialsubs = MSocialSubscription.objects.filter(user_id=request.user.pk)
for subtype in [feeds, socialsubs]:
for sub in subtype:
if days == 0:
sub.mark_feed_read()
else:
if sub.mark_read_date < read_date:
sub.needs_unread_recalc = True
sub.mark_read_date = read_date
sub.save()
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:feeds')
logging.user(request, "~FMMarking %s as read: ~SB%s days" % (("all" if not infrequent else "infrequent stories"), days,))
return dict(code=code)
@ajax_login_required
@json.json_view
def mark_story_as_read(request):
story_ids = request.POST.getlist('story_id') or request.POST.getlist('story_id[]')
try:
feed_id = int(get_argument_or_404(request, 'feed_id'))
except ValueError:
return dict(code=-1, errors=["You must pass a valid feed_id: %s" %
request.POST.get('feed_id')])
try:
usersub = UserSubscription.objects.select_related('feed').get(user=request.user, feed=feed_id)
except Feed.DoesNotExist:
duplicate_feed = DuplicateFeed.objects.filter(duplicate_feed_id=feed_id)
if duplicate_feed:
feed_id = duplicate_feed[0].feed_id
try:
usersub = UserSubscription.objects.get(user=request.user,
feed=duplicate_feed[0].feed)
except (Feed.DoesNotExist):
return dict(code=-1, errors=["No feed exists for feed_id %d." % feed_id])
else:
return dict(code=-1, errors=["No feed exists for feed_id %d." % feed_id])
except UserSubscription.DoesNotExist:
usersub = None
if usersub:
data = usersub.mark_story_ids_as_read(story_ids, request=request)
else:
data = dict(code=-1, errors=["User is not subscribed to this feed."])
return data
@ajax_login_required
@json.json_view
def mark_story_hashes_as_read(request):
retrying_failed = is_true(request.POST.get('retrying_failed', False))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
try:
story_hashes = request.POST.getlist('story_hash') or request.POST.getlist('story_hash[]')
except UnreadablePostError:
return dict(code=-1, message="Missing `story_hash` list parameter.")
feed_ids, friend_ids = RUserStory.mark_story_hashes_read(request.user.pk, story_hashes, username=request.user.username)
if request.user.profile.is_archive:
RUserUnreadStory.mark_read(request.user.pk, story_hashes)
if friend_ids:
socialsubs = MSocialSubscription.objects.filter(
user_id=request.user.pk,
subscription_user_id__in=friend_ids)
for socialsub in socialsubs:
if not socialsub.needs_unread_recalc:
socialsub.needs_unread_recalc = True
socialsub.save()
r.publish(request.user.username, 'social:%s' % socialsub.subscription_user_id)
# Also count on original subscription
for feed_id in feed_ids:
usersubs = UserSubscription.objects.filter(user=request.user.pk, feed=feed_id)
if usersubs:
usersub = usersubs[0]
usersub.last_read_date = datetime.datetime.now()
if not usersub.needs_unread_recalc:
usersub.needs_unread_recalc = True
usersub.save(update_fields=['needs_unread_recalc', 'last_read_date'])
else:
usersub.save(update_fields=['last_read_date'])
r.publish(request.user.username, 'feed:%s' % feed_id)
hash_count = len(story_hashes)
logging.user(request, "~FYRead %s %s: %s %s" % (
hash_count, 'story' if hash_count == 1 else 'stories',
story_hashes,
'(retrying failed)' if retrying_failed else ''))
return dict(code=1, story_hashes=story_hashes,
feed_ids=feed_ids, friend_user_ids=friend_ids)
@ajax_login_required
@json.json_view
def mark_feed_stories_as_read(request):
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
feeds_stories = request.POST.get('feeds_stories', "{}")
feeds_stories = json.decode(feeds_stories)
data = {
'code': -1,
'message': 'Nothing was marked as read'
}
for feed_id, story_ids in list(feeds_stories.items()):
try:
feed_id = int(feed_id)
except ValueError:
continue
try:
usersub = UserSubscription.objects.select_related('feed').get(user=request.user, feed=feed_id)
data = usersub.mark_story_ids_as_read(story_ids, request=request)
except UserSubscription.DoesNotExist:
return dict(code=-1, error="You are not subscribed to this feed_id: %d" % feed_id)
except Feed.DoesNotExist:
duplicate_feed = DuplicateFeed.objects.filter(duplicate_feed_id=feed_id)
try:
if not duplicate_feed: raise Feed.DoesNotExist
usersub = UserSubscription.objects.get(user=request.user,
feed=duplicate_feed[0].feed)
data = usersub.mark_story_ids_as_read(story_ids, request=request)
except (UserSubscription.DoesNotExist, Feed.DoesNotExist):
return dict(code=-1, error="No feed exists for feed_id: %d" % feed_id)
r.publish(request.user.username, 'feed:%s' % feed_id)
return data
@ajax_login_required
@json.json_view
def mark_social_stories_as_read(request):
code = 1
errors = []
data = {}
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
users_feeds_stories = request.POST.get('users_feeds_stories', "{}")
users_feeds_stories = json.decode(users_feeds_stories)
for social_user_id, feeds in list(users_feeds_stories.items()):
for feed_id, story_ids in list(feeds.items()):
feed_id = int(feed_id)
try:
socialsub = MSocialSubscription.objects.get(user_id=request.user.pk,
subscription_user_id=social_user_id)
data = socialsub.mark_story_ids_as_read(story_ids, feed_id, request=request)
except OperationError as e:
code = -1
errors.append("Already read story: %s" % e)
except MSocialSubscription.DoesNotExist:
MSocialSubscription.mark_unsub_story_ids_as_read(request.user.pk, social_user_id,
story_ids, feed_id,
request=request)
except Feed.DoesNotExist:
duplicate_feed = DuplicateFeed.objects.filter(duplicate_feed_id=feed_id)
if duplicate_feed:
try:
socialsub = MSocialSubscription.objects.get(user_id=request.user.pk,
subscription_user_id=social_user_id)
data = socialsub.mark_story_ids_as_read(story_ids, duplicate_feed[0].feed.pk, request=request)
except (UserSubscription.DoesNotExist, Feed.DoesNotExist):
code = -1
errors.append("No feed exists for feed_id %d." % feed_id)
else:
continue
r.publish(request.user.username, 'feed:%s' % feed_id)
r.publish(request.user.username, 'social:%s' % social_user_id)
data.update(code=code, errors=errors)
return data
@required_params('story_id', feed_id=int)
@ajax_login_required
@json.json_view
def mark_story_as_unread(request):
story_id = request.POST.get('story_id', None)
feed_id = int(request.POST.get('feed_id', 0))
try:
usersub = UserSubscription.objects.select_related('feed').get(user=request.user, feed=feed_id)
feed = usersub.feed
except UserSubscription.DoesNotExist:
usersub = None
feed = Feed.get_by_id(feed_id)
if usersub and not usersub.needs_unread_recalc:
usersub.needs_unread_recalc = True
usersub.save(update_fields=['needs_unread_recalc'])
data = dict(code=0, payload=dict(story_id=story_id))
story, found_original = MStory.find_story(feed_id, story_id)
if not story:
logging.user(request, "~FY~SBUnread~SN story in feed: %s (NOT FOUND)" % (feed))
return dict(code=-1, message="Story not found.")
message = RUserStory.story_can_be_marked_unread_by_user(story, request.user)
if message:
data['code'] = -1
data['message'] = message
return data
if usersub:
data = usersub.invert_read_stories_after_unread_story(story, request)
social_subs = MSocialSubscription.mark_dirty_sharing_story(user_id=request.user.pk,
story_feed_id=feed_id,
story_guid_hash=story.guid_hash)
dirty_count = social_subs and social_subs.count()
dirty_count = ("(%s social_subs)" % dirty_count) if dirty_count else ""
RUserStory.mark_story_hash_unread(request.user, story_hash=story.story_hash)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'feed:%s' % feed_id)
logging.user(request, "~FY~SBUnread~SN story in feed: %s %s" % (feed, dirty_count))
return data
@ajax_login_required
@json.json_view
@required_params('story_hash')
def mark_story_hash_as_unread(request):
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
story_hashes = request.POST.getlist('story_hash') or request.POST.getlist('story_hash[]')
is_list = len(story_hashes) > 1
datas = []
for story_hash in story_hashes:
feed_id, _ = MStory.split_story_hash(story_hash)
story, _ = MStory.find_story(feed_id, story_hash)
if not story:
data = dict(code=-1, message="That story has been removed from the feed, no need to mark it unread.", story_hash=story_hash)
if not is_list:
return data
else:
datas.append(data)
message = RUserStory.story_can_be_marked_unread_by_user(story, request.user)
if message:
data = dict(code=-1, message=message, story_hash=story_hash)
if not is_list:
return data
else:
datas.append(data)
# Also count on original subscription
usersubs = UserSubscription.objects.filter(user=request.user.pk, feed=feed_id)
if usersubs:
usersub = usersubs[0]
if not usersub.needs_unread_recalc:
usersub.needs_unread_recalc = True
usersub.save(update_fields=['needs_unread_recalc'])
data = usersub.invert_read_stories_after_unread_story(story, request)
r.publish(request.user.username, 'feed:%s' % feed_id)
feed_id, friend_ids = RUserStory.mark_story_hash_unread(request.user, story_hash)
if friend_ids:
socialsubs = MSocialSubscription.objects.filter(
user_id=request.user.pk,
subscription_user_id__in=friend_ids)
for socialsub in socialsubs:
if not socialsub.needs_unread_recalc:
socialsub.needs_unread_recalc = True
socialsub.save()
r.publish(request.user.username, 'social:%s' % socialsub.subscription_user_id)
logging.user(request, "~FYUnread story in feed/socialsubs: %s/%s" % (feed_id, friend_ids))
data = dict(code=1, story_hash=story_hash, feed_id=feed_id, friend_user_ids=friend_ids)
if not is_list:
return data
else:
datas.append(data)
return datas
@ajax_login_required
@json.json_view
def mark_feed_as_read(request):
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
feed_ids = request.POST.getlist('feed_id') or request.POST.getlist('feed_id[]')
cutoff_timestamp = int(request.POST.get('cutoff_timestamp', 0))
direction = request.POST.get('direction', 'older')
infrequent = is_true(request.POST.get('infrequent', False))
if infrequent:
infrequent = request.POST.get('infrequent')
multiple = len(feed_ids) > 1
code = 1
errors = []
cutoff_date = datetime.datetime.fromtimestamp(cutoff_timestamp) if cutoff_timestamp else None
if infrequent:
feed_ids = Feed.low_volume_feeds(feed_ids, stories_per_month=infrequent)
feed_ids = [str(f) for f in feed_ids] # This method expects strings
if cutoff_date:
logging.user(request, "~FMMark %s feeds read, %s - cutoff: %s/%s" %
(len(feed_ids), direction, cutoff_timestamp, cutoff_date))
for feed_id in feed_ids:
if 'social:' in feed_id:
user_id = int(feed_id.replace('social:', ''))
try:
sub = MSocialSubscription.objects.get(user_id=request.user.pk,
subscription_user_id=user_id)
except MSocialSubscription.DoesNotExist:
logging.user(request, "~FRCouldn't find socialsub: %s" % user_id)
continue
if not multiple:
sub_user = User.objects.get(pk=sub.subscription_user_id)
logging.user(request, "~FMMarking social feed as read: ~SB%s" % (sub_user.username,))
else:
try:
feed = Feed.objects.get(id=feed_id)
sub = UserSubscription.objects.get(feed=feed, user=request.user)
if not multiple:
logging.user(request, "~FMMarking feed as read: ~SB%s" % (feed,))
except (Feed.DoesNotExist, UserSubscription.DoesNotExist) as e:
errors.append("User not subscribed: %s" % e)
continue
except (ValueError) as e:
errors.append("Invalid feed_id: %s" % e)
continue
if not sub:
errors.append("User not subscribed: %s" % feed_id)
continue
try:
if direction == "older":
marked_read = sub.mark_feed_read(cutoff_date=cutoff_date)
else:
marked_read = sub.mark_newer_stories_read(cutoff_date=cutoff_date)
if marked_read and not multiple:
r.publish(request.user.username, 'feed:%s' % feed_id)
except IntegrityError as e:
errors.append("Could not mark feed as read: %s" % e)
code = -1
if multiple:
logging.user(request, "~FMMarking ~SB%s~SN feeds as read" % len(feed_ids))
r.publish(request.user.username, 'refresh:%s' % ','.join(feed_ids))
if errors:
logging.user(request, "~FMMarking read had errors: ~FR%s" % errors)
return dict(code=code, errors=errors, cutoff_date=cutoff_date, direction=direction)
def _parse_user_info(user):
return {
'user_info': {
'is_anonymous': json.encode(user.is_anonymous),
'is_authenticated': json.encode(user.is_authenticated),
'username': json.encode(user.username if user.is_authenticated else 'Anonymous')
}
}
@ajax_login_required
@json.json_view
def add_url(request):
code = 0
url = request.POST['url']
folder = request.POST.get('folder', '').replace('river:', '')
new_folder = request.POST.get('new_folder', '').replace('river:', '')
auto_active = is_true(request.POST.get('auto_active', 1))
skip_fetch = is_true(request.POST.get('skip_fetch', False))
feed = None
if not url:
code = -1
message = 'Enter in the website address or the feed URL.'
elif any([(banned_url in url) for banned_url in BANNED_URLS]):
code = -1
message = "The publisher of this website has banned NewsBlur."
elif re.match('(https?://)?twitter.com/\w+/?$', url):
if not request.user.profile.is_premium:
message = "You must be a premium subscriber to add Twitter feeds."
code = -1
else:
# Check if Twitter API is active for user
ss = MSocialServices.get_user(request.user.pk)
try:
if not ss.twitter_uid:
raise tweepy.TweepError("No API token")
ss.twitter_api().me()
except tweepy.TweepError:
code = -1
message = "Your Twitter connection isn't setup. Go to Manage - Friends/Followers and reconnect Twitter."
if code == -1:
return dict(code=code, message=message)
if new_folder:
usf, _ = UserSubscriptionFolders.objects.get_or_create(user=request.user)
usf.add_folder(folder, new_folder)
folder = new_folder
code, message, us = UserSubscription.add_subscription(user=request.user, feed_address=url,
folder=folder, auto_active=auto_active,
skip_fetch=skip_fetch)
feed = us and us.feed
if feed:
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:%s' % feed.pk)
MUserSearch.schedule_index_feeds_for_search(feed.pk, request.user.pk)
return dict(code=code, message=message, feed=feed)
@ajax_login_required
@json.json_view
def add_folder(request):
folder = request.POST['folder'].replace('river:', '')
parent_folder = request.POST.get('parent_folder', '').replace('river:', '')
folders = None
logging.user(request, "~FRAdding Folder: ~SB%s (in %s)" % (folder, parent_folder))
if folder:
code = 1
message = ""
user_sub_folders_object, _ = UserSubscriptionFolders.objects.get_or_create(user=request.user)
user_sub_folders_object.add_folder(parent_folder, folder)
folders = json.decode(user_sub_folders_object.folders)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:feeds')
else:
code = -1
message = "Gotta write in a folder name."
return dict(code=code, message=message, folders=folders)
@ajax_login_required
@json.json_view
def delete_feed(request):
feed_id = int(request.POST['feed_id'])
in_folder = request.POST.get('in_folder', '').replace('river:', '')
if not in_folder or in_folder == ' ':
in_folder = ""
user_sub_folders = get_object_or_404(UserSubscriptionFolders, user=request.user)
user_sub_folders.delete_feed(feed_id, in_folder)
feed = Feed.objects.filter(pk=feed_id)
if feed:
feed[0].count_subscribers()
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:feeds')
return dict(code=1, message="Removed %s from '%s'." % (feed, in_folder))
@ajax_login_required
@json.json_view
def delete_feed_by_url(request):
message = ""
code = 0
url = request.POST['url']
in_folder = request.POST.get('in_folder', '').replace('river:', '')
if in_folder == ' ':
in_folder = ""
logging.user(request.user, "~FBFinding feed (delete_feed_by_url): %s" % url)
feed = Feed.get_feed_from_url(url, create=False)
if feed:
user_sub_folders = get_object_or_404(UserSubscriptionFolders, user=request.user)
user_sub_folders.delete_feed(feed.pk, in_folder)
code = 1
feed = Feed.objects.filter(pk=feed.pk)
if feed:
feed[0].count_subscribers()
else:
code = -1
message = "URL not found."
return dict(code=code, message=message)
@ajax_login_required
@json.json_view
def delete_folder(request):
folder_to_delete = request.POST.get('folder_name') or request.POST.get('folder_to_delete')
in_folder = request.POST.get('in_folder', None)
feed_ids_in_folder = request.POST.getlist('feed_id') or request.POST.getlist('feed_id[]')
feed_ids_in_folder = [int(f) for f in feed_ids_in_folder if f]
request.user.profile.send_opml_export_email(reason="You have deleted an entire folder of feeds, so here's a backup of all of your subscriptions just in case.")
# Works piss poor with duplicate folder titles, if they are both in the same folder.
# Deletes all, but only in the same folder parent. But nobody should be doing that, right?
user_sub_folders = get_object_or_404(UserSubscriptionFolders, user=request.user)
user_sub_folders.delete_folder(folder_to_delete, in_folder, feed_ids_in_folder)
folders = json.decode(user_sub_folders.folders)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:feeds')
return dict(code=1, folders=folders)
@required_params('feeds_by_folder')
@ajax_login_required
@json.json_view
def delete_feeds_by_folder(request):
feeds_by_folder = json.decode(request.POST['feeds_by_folder'])
request.user.profile.send_opml_export_email(reason="You have deleted a number of feeds at once, so here's a backup of all of your subscriptions just in case.")
# Works piss poor with duplicate folder titles, if they are both in the same folder.
# Deletes all, but only in the same folder parent. But nobody should be doing that, right?
user_sub_folders = get_object_or_404(UserSubscriptionFolders, user=request.user)
user_sub_folders.delete_feeds_by_folder(feeds_by_folder)
folders = json.decode(user_sub_folders.folders)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:feeds')
return dict(code=1, folders=folders)
@ajax_login_required
@json.json_view
def rename_feed(request):
feed = get_object_or_404(Feed, pk=int(request.POST['feed_id']))
try:
user_sub = UserSubscription.objects.get(user=request.user, feed=feed)
except UserSubscription.DoesNotExist:
return dict(code=-1, message=f"You are not subscribed to {feed.feed_title}")
feed_title = request.POST['feed_title']
logging.user(request, "~FRRenaming feed '~SB%s~SN' to: ~SB%s" % (
feed.feed_title, feed_title))
user_sub.user_title = feed_title
user_sub.save()
return dict(code=1)
@ajax_login_required
@json.json_view
def rename_folder(request):
folder_to_rename = request.POST.get('folder_name') or request.POST.get('folder_to_rename')
new_folder_name = request.POST['new_folder_name']
in_folder = request.POST.get('in_folder', '').replace('river:', '')
if 'Top Level' in in_folder: in_folder = ''
code = 0
# Works piss poor with duplicate folder titles, if they are both in the same folder.
# renames all, but only in the same folder parent. But nobody should be doing that, right?
if folder_to_rename and new_folder_name:
user_sub_folders = get_object_or_404(UserSubscriptionFolders, user=request.user)
user_sub_folders.rename_folder(folder_to_rename, new_folder_name, in_folder)
code = 1
else:
code = -1
return dict(code=code)
@ajax_login_required
@json.json_view
def move_feed_to_folders(request):
feed_id = int(request.POST['feed_id'])
in_folders = request.POST.getlist('in_folders', '') or request.POST.getlist('in_folders[]', '')
to_folders = request.POST.getlist('to_folders', '') or request.POST.getlist('to_folders[]', '')
user_sub_folders = get_object_or_404(UserSubscriptionFolders, user=request.user)
user_sub_folders = user_sub_folders.move_feed_to_folders(feed_id, in_folders=in_folders,
to_folders=to_folders)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:feeds')
return dict(code=1, folders=json.decode(user_sub_folders.folders))
@ajax_login_required
@json.json_view
def move_feed_to_folder(request):
feed_id = int(request.POST['feed_id'])
in_folder = request.POST.get('in_folder', '')
to_folder = request.POST.get('to_folder', '')
user_sub_folders = get_object_or_404(UserSubscriptionFolders, user=request.user)
user_sub_folders = user_sub_folders.move_feed_to_folder(feed_id, in_folder=in_folder,
to_folder=to_folder)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:feeds')
return dict(code=1, folders=json.decode(user_sub_folders.folders))
@ajax_login_required
@json.json_view
def move_folder_to_folder(request):
folder_name = request.POST['folder_name']
in_folder = request.POST.get('in_folder', '')
to_folder = request.POST.get('to_folder', '')
user_sub_folders = get_object_or_404(UserSubscriptionFolders, user=request.user)
user_sub_folders = user_sub_folders.move_folder_to_folder(folder_name, in_folder=in_folder, to_folder=to_folder)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:feeds')
return dict(code=1, folders=json.decode(user_sub_folders.folders))
@required_params('feeds_by_folder', 'to_folder')
@ajax_login_required
@json.json_view
def move_feeds_by_folder_to_folder(request):
feeds_by_folder = json.decode(request.POST['feeds_by_folder'])
to_folder = request.POST['to_folder']
new_folder = request.POST.get('new_folder', None)
request.user.profile.send_opml_export_email(reason="You have moved a number of feeds at once, so here's a backup of all of your subscriptions just in case.")
user_sub_folders = get_object_or_404(UserSubscriptionFolders, user=request.user)
if new_folder:
user_sub_folders.add_folder(to_folder, new_folder)
to_folder = new_folder
user_sub_folders = user_sub_folders.move_feeds_by_folder_to_folder(feeds_by_folder, to_folder)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:feeds')
return dict(code=1, folders=json.decode(user_sub_folders.folders))
@login_required
def add_feature(request):
if not request.user.is_staff:
return HttpResponseForbidden()
code = -1
form = FeatureForm(request.POST)
if form.is_valid():
form.save()
code = 1
return HttpResponseRedirect(reverse('index'))
return dict(code=code)
@json.json_view
def load_features(request):
user = get_user(request)
page = max(int(request.GET.get('page', 0)), 0)
if page > 1:
logging.user(request, "~FBBrowse features: ~SBPage #%s" % (page+1))
features = list(Feature.objects.all()[page*3:(page+1)*3+1].values())
features = [{
'description': f['description'],
'date': localtime_for_timezone(f['date'], user.profile.timezone).strftime("%b %d, %Y")
} for f in features]
return features
@ajax_login_required
@json.json_view
def save_feed_order(request):
folders = request.POST.get('folders')
if folders:
# Test that folders can be JSON decoded
folders_list = json.decode(folders)
assert folders_list is not None
logging.user(request, "~FBFeed re-ordering: ~SB%s folders/feeds" % (len(folders_list)))
user_sub_folders = UserSubscriptionFolders.objects.get(user=request.user)
user_sub_folders.folders = folders
user_sub_folders.save()
return {}
@json.json_view
def feeds_trainer(request):
classifiers = []
feed_id = request.GET.get('feed_id')
user = get_user(request)
usersubs = UserSubscription.objects.filter(user=user, active=True)
if feed_id:
feed = get_object_or_404(Feed, pk=feed_id)
usersubs = usersubs.filter(feed=feed)
usersubs = usersubs.select_related('feed').order_by('-feed__stories_last_month')
for us in usersubs:
if (not us.is_trained and us.feed.stories_last_month > 0) or feed_id:
classifier = dict()
classifier['classifiers'] = get_classifiers_for_user(user, feed_id=us.feed.pk)
classifier['feed_id'] = us.feed_id
classifier['stories_last_month'] = us.feed.stories_last_month
classifier['num_subscribers'] = us.feed.num_subscribers
classifier['feed_tags'] = json.decode(us.feed.data.popular_tags) if us.feed.data.popular_tags else []
classifier['feed_authors'] = json.decode(us.feed.data.popular_authors) if us.feed.data.popular_authors else []
classifiers.append(classifier)
user.profile.has_trained_intelligence = True
user.profile.save()
logging.user(user, "~FGLoading Trainer: ~SB%s feeds" % (len(classifiers)))
return classifiers
@ajax_login_required
@json.json_view
def save_feed_chooser(request):
is_premium = request.user.profile.is_premium
approved_feeds = request.POST.getlist('approved_feeds') or request.POST.getlist('approved_feeds[]')
approved_feeds = [int(feed_id) for feed_id in approved_feeds if feed_id]
approve_all = False
if not is_premium:
approved_feeds = approved_feeds[:64]
elif is_premium and not approved_feeds:
approve_all = True
activated = 0
usersubs = UserSubscription.objects.filter(user=request.user)
for sub in usersubs:
try:
if sub.feed_id in approved_feeds or approve_all:
activated += 1
if not sub.active:
sub.active = True
sub.save()
if sub.feed.active_subscribers <= 0:
sub.feed.count_subscribers()
elif sub.active:
sub.active = False
sub.save()
except Feed.DoesNotExist:
pass
UserSubscription.queue_new_feeds(request.user)
UserSubscription.refresh_stale_feeds(request.user, exclude_new=True)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'reload:feeds')
logging.user(request, "~BB~FW~SBFeed chooser: ~FC%s~SN/~SB%s" % (
activated,
usersubs.count()
))
return {'activated': activated}
@ajax_login_required
def retrain_all_sites(request):
for sub in UserSubscription.objects.filter(user=request.user):
sub.is_trained = False
sub.save()
return feeds_trainer(request)
@login_required
def activate_premium_account(request):
try:
usersubs = UserSubscription.objects.select_related('feed').filter(user=request.user)
for sub in usersubs:
sub.active = True
sub.save()
if sub.feed.premium_subscribers <= 0:
sub.feed.count_subscribers()
sub.feed.schedule_feed_fetch_immediately()
except Exception as e:
logging.user(request, "~BR~FWPremium activation failed: {e} {usersubs}")
request.user.profile.is_premium = True
request.user.profile.save()
return HttpResponseRedirect(reverse('index'))
@login_required
def login_as(request):
if not request.user.is_staff:
logging.user(request, "~SKNON-STAFF LOGGING IN AS ANOTHER USER!")
assert False
return HttpResponseForbidden()
username = request.GET['user']
user = get_object_or_404(User, username__iexact=username)
user.backend = settings.AUTHENTICATION_BACKENDS[0]
login_user(request, user, backend='django.contrib.auth.backends.ModelBackend')
return HttpResponseRedirect(reverse('index'))
def iframe_buster(request):
logging.user(request, "~FB~SBiFrame bust!")
return HttpResponse(status=204)
@required_params('story_id', feed_id=int)
@ajax_login_required
@json.json_view
def mark_story_as_starred(request):
return _mark_story_as_starred(request)
@required_params('story_hash')
@ajax_login_required
@json.json_view
def mark_story_hash_as_starred(request):
return _mark_story_as_starred(request)
def _mark_story_as_starred(request):
code = 1
feed_id = int(request.POST.get('feed_id', 0))
story_id = request.POST.get('story_id', None)
user_tags = request.POST.getlist('user_tags') or request.POST.getlist('user_tags[]')
user_notes = request.POST.get('user_notes', None)
highlights = request.POST.getlist('highlights') or request.POST.getlist('highlights[]') or []
message = ""
story_hashes = request.POST.getlist('story_hash') or request.POST.getlist('story_hash[]')
is_list = len(story_hashes) > 1
datas = []
if not len(story_hashes):
story, _ = MStory.find_story(story_feed_id=feed_id, story_id=story_id)
if story:
story_hashes = [story.story_hash]
if not len(story_hashes):
return {'code': -1, 'message': "Could not find story to save."}
for story_hash in story_hashes:
story, _ = MStory.find_story(story_hash=story_hash)
if not story:
logging.user(request, "~FCStarring ~FRfailed~FC: %s not found" % (story_hash))
datas.append({'code': -1, 'message': "Could not save story, not found", 'story_hash': story_hash})
continue
feed_id = story and story.story_feed_id
story_db = dict([(k, v) for k, v in list(story._data.items())
if k is not None and v is not None])
# Pop all existing user-specific fields because we don't want to reuse them from the found story
# in case MStory.find_story uses somebody else's saved/shared story (because the original is deleted)
story_db.pop('user_id', None)
story_db.pop('starred_date', None)
story_db.pop('id', None)
story_db.pop('user_tags', None)
story_db.pop('highlights', None)
story_db.pop('user_notes', None)
now = datetime.datetime.now()
story_values = dict(starred_date=now, user_tags=user_tags, highlights=highlights, user_notes=user_notes, **story_db)
params = dict(story_guid=story.story_guid, user_id=request.user.pk)
starred_story = MStarredStory.objects(**params).limit(1)
created = False
changed_user_notes = False
removed_user_tags = []
removed_highlights = []
if not starred_story:
params.update(story_values)
if 'story_latest_content_z' in params:
params.pop('story_latest_content_z')
try:
starred_story = MStarredStory.objects.create(**params)
except OperationError as e:
logging.user(request, "~FCStarring ~FRfailed~FC: ~SB%s (~FM~SB%s~FC~SN)" % (story.story_title[:32], e))
datas.append({'code': -1, 'message': "Could not save story due to: %s" % e, 'story_hash': story_hash})
created = True
MActivity.new_starred_story(user_id=request.user.pk,
story_title=story.story_title,
story_feed_id=feed_id,
story_id=starred_story.story_guid)
new_user_tags = user_tags
new_highlights = highlights
changed_user_notes = bool(user_notes)
MStarredStoryCounts.adjust_count(request.user.pk, feed_id=feed_id, amount=1)
else:
starred_story = starred_story[0]
new_user_tags = list(set(user_tags) - set(starred_story.user_tags or []))
removed_user_tags = list(set(starred_story.user_tags or []) - set(user_tags))
new_highlights = list(set(highlights) - set(starred_story.highlights or []))
removed_highlights = list(set(starred_story.highlights or []) - set(highlights))
changed_user_notes = bool(user_notes != starred_story.user_notes)
starred_story.user_tags = user_tags
starred_story.highlights = highlights
starred_story.user_notes = user_notes
starred_story.save()
if len(highlights) == 1 and len(new_highlights) == 1:
MStarredStoryCounts.adjust_count(request.user.pk, highlights=True, amount=1)
elif len(highlights) == 0 and len(removed_highlights):
MStarredStoryCounts.adjust_count(request.user.pk, highlights=True, amount=-1)
for tag in new_user_tags:
MStarredStoryCounts.adjust_count(request.user.pk, tag=tag, amount=1)
for tag in removed_user_tags:
MStarredStoryCounts.adjust_count(request.user.pk, tag=tag, amount=-1)
if random.random() < 0.01:
MStarredStoryCounts.schedule_count_tags_for_user(request.user.pk)
MStarredStoryCounts.count_for_user(request.user.pk, total_only=True)
starred_counts, starred_count = MStarredStoryCounts.user_counts(request.user.pk, include_total=True)
if not starred_count and len(starred_counts):
starred_count = MStarredStory.objects(user_id=request.user.pk).count()
if not changed_user_notes:
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'story:starred:%s' % story.story_hash)
if created:
logging.user(request, "~FCStarring: ~SB%s (~FM~SB%s~FC~SN)" % (story.story_title[:32], starred_story.user_tags))
else:
logging.user(request, "~FCUpdating starred:~SN~FC ~SB%s~SN (~FM~SB%s~FC~SN/~FM%s~FC)" % (story.story_title[:32], starred_story.user_tags, starred_story.user_notes))
datas.append({'code': code, 'message': message, 'starred_count': starred_count, 'starred_counts': starred_counts})
if len(datas) >= 2:
return datas
elif len(datas) == 1:
return datas[0]
return datas
@required_params('story_id')
@ajax_login_required
@json.json_view
def mark_story_as_unstarred(request):
return _mark_story_as_unstarred(request)
@required_params('story_hash')
@ajax_login_required
@json.json_view
def mark_story_hash_as_unstarred(request):
return _mark_story_as_unstarred(request)
def _mark_story_as_unstarred(request):
code = 1
story_id = request.POST.get('story_id', None)
story_hashes = request.POST.getlist('story_hash') or request.POST.getlist('story_hash[]')
starred_counts = None
starred_story = None
if story_id:
starred_story = MStarredStory.objects(user_id=request.user.pk, story_guid=story_id)
if starred_story:
starred_story = starred_story[0]
story_hashes = [starred_story.story_hash]
else:
story_hashes = [story_id]
datas = []
for story_hash in story_hashes:
starred_story = MStarredStory.objects(user_id=request.user.pk, story_hash=story_hash)
if not starred_story:
logging.user(request, "~FCUnstarring ~FRfailed~FC: %s not found" % (story_hash))
datas.append({'code': -1, 'message': "Could not unsave story, not found", 'story_hash': story_hash})
continue
starred_story = starred_story[0]
logging.user(request, "~FCUnstarring: ~SB%s" % (starred_story.story_title[:50]))
user_tags = starred_story.user_tags
feed_id = starred_story.story_feed_id
MActivity.remove_starred_story(user_id=request.user.pk,
story_feed_id=starred_story.story_feed_id,
story_id=starred_story.story_guid)
starred_story.user_id = 0
try:
starred_story.save()
except NotUniqueError:
starred_story.delete()
MStarredStoryCounts.adjust_count(request.user.pk, feed_id=feed_id, amount=-1)
for tag in user_tags:
try:
MStarredStoryCounts.adjust_count(request.user.pk, tag=tag, amount=-1)
except MStarredStoryCounts.DoesNotExist:
pass
MStarredStoryCounts.schedule_count_tags_for_user(request.user.pk)
MStarredStoryCounts.count_for_user(request.user.pk, total_only=True)
starred_counts = MStarredStoryCounts.user_counts(request.user.pk)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(request.user.username, 'story:unstarred:%s' % starred_story.story_hash)
if not story_hashes:
datas.append(dict(code=-1, message=f"Failed to find {story_hashes}"))
return {'code': code, 'starred_counts': starred_counts, 'messages': datas}
@ajax_login_required
@json.json_view
def starred_counts(request):
starred_counts, starred_count = MStarredStoryCounts.user_counts(request.user.pk, include_total=True)
logging.user(request, "~FCRequesting starred counts: ~SB%s stories (%s tags)" % (starred_count, len([s for s in starred_counts if s['tag']])))
return {'starred_count': starred_count, 'starred_counts': starred_counts}
@ajax_login_required
@json.json_view
def send_story_email(request):
def validate_email_as_bool(email):
try:
validate_email(email)
return True
except:
return False
code = 1
message = 'OK'
user = get_user(request)
story_id = request.POST['story_id']
feed_id = request.POST['feed_id']
to_addresses = request.POST.get('to', '').replace(',', ' ').replace(' ', ' ').strip().split(' ')
from_name = request.POST['from_name']
from_email = request.POST['from_email']
email_cc = is_true(request.POST.get('email_cc', 'true'))
comments = request.POST['comments']
comments = comments[:2048] # Separated due to PyLint
from_address = 'share@newsblur.com'
share_user_profile = MSocialProfile.get_user(request.user.pk)
quota = 32 if user.profile.is_premium else 1
if share_user_profile.over_story_email_quota(quota=quota):
code = -1
if user.profile.is_premium:
message = 'You can only send %s stories per day by email.' % quota
else:
message = 'Upgrade to a premium subscription to send more than one story per day by email.'
logging.user(request, '~BRNOT ~BMSharing story by email to %s recipient, over quota: %s/%s' %
(len(to_addresses), story_id, feed_id))
elif not to_addresses:
code = -1
message = 'Please provide at least one email address.'
elif not all(validate_email_as_bool(to_address) for to_address in to_addresses if to_addresses):
code = -1
message = 'You need to send the email to a valid email address.'
elif not validate_email_as_bool(from_email):
code = -1
message = 'You need to provide your email address.'
elif not from_name:
code = -1
message = 'You need to provide your name.'
else:
story, _ = MStory.find_story(feed_id, story_id)
story = Feed.format_story(story, feed_id, text=True)
feed = Feed.get_by_id(story['story_feed_id'])
params = {
"to_addresses": to_addresses,
"from_name": from_name,
"from_email": from_email,
"email_cc": email_cc,
"comments": comments,
"from_address": from_address,
"story": story,
"feed": feed,
"share_user_profile": share_user_profile,
}
text = render_to_string('mail/email_story.txt', params)
html = render_to_string('mail/email_story.xhtml', params)
subject = '%s' % (story['story_title'])
cc = None
if email_cc:
cc = ['%s <%s>' % (from_name, from_email)]
subject = subject.replace('\n', ' ')
msg = EmailMultiAlternatives(subject, text,
from_email='NewsBlur <%s>' % from_address,
to=to_addresses,
cc=cc,
headers={'Reply-To': "%s <%s>" % (from_name, from_email)})
msg.attach_alternative(html, "text/html")
# try:
msg.send()
# except boto.ses.connection.BotoServerError as e:
# code = -1
# message = "Email error: %s" % str(e)
share_user_profile.save_sent_email()
logging.user(request, '~BMSharing story by email to %s recipient%s (%s): ~FY~SB%s~SN~BM~FY/~SB%s' %
(len(to_addresses), '' if len(to_addresses) == 1 else 's', to_addresses,
story['story_title'][:50], feed and feed.feed_title[:50]))
return {'code': code, 'message': message}
@json.json_view
def load_tutorial(request):
if request.GET.get('finished'):
logging.user(request, '~BY~FW~SBFinishing Tutorial')
return {}
else:
newsblur_feed = Feed.objects.filter(feed_address__icontains='blog.newsblur.com').order_by('-pk')[0]
logging.user(request, '~BY~FW~SBLoading Tutorial')
return {
'newsblur_feed': newsblur_feed.canonical()
}
@required_params('query', 'feed_id')
@json.json_view
def save_search(request):
feed_id = request.POST['feed_id']
query = request.POST['query']
MSavedSearch.save_search(user_id=request.user.pk, feed_id=feed_id, query=query)
saved_searches = MSavedSearch.user_searches(request.user.pk)
return {
'saved_searches': saved_searches,
}
@required_params('query', 'feed_id')
@json.json_view
def delete_search(request):
feed_id = request.POST['feed_id']
query = request.POST['query']
MSavedSearch.delete_search(user_id=request.user.pk, feed_id=feed_id, query=query)
saved_searches = MSavedSearch.user_searches(request.user.pk)
return {
'saved_searches': saved_searches,
}
@required_params('river_id', 'river_side', 'river_order')
@json.json_view
def save_dashboard_river(request):
river_id = request.POST['river_id']
river_side = request.POST['river_side']
river_order = int(request.POST['river_order'])
logging.user(request, "~FCSaving dashboard river: ~SB%s~SN (%s %s)" % (river_id, river_side, river_order))
MDashboardRiver.save_user(request.user.pk, river_id, river_side, river_order)
dashboard_rivers = MDashboardRiver.get_user_rivers(request.user.pk)
return {
'dashboard_rivers': dashboard_rivers,
}
@required_params('river_id', 'river_side', 'river_order')
@json.json_view
def remove_dashboard_river(request):
river_id = request.POST['river_id']
river_side = request.POST['river_side']
river_order = int(request.POST['river_order'])
logging.user(request, "~FRRemoving~FC dashboard river: ~SB%s~SN (%s %s)" % (river_id, river_side, river_order))
MDashboardRiver.remove_river(request.user.pk, river_side, river_order)
dashboard_rivers = MDashboardRiver.get_user_rivers(request.user.pk)
return {
'dashboard_rivers': dashboard_rivers,
}