public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-09-18 21:43:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into hidden_stories

Browse source 
* master:
  Handling broken getsatisfaction feedback url.
  Handling feeds that go missing.
  Fixing yet another requests error on icon fetching.
  Handling more errors when fetching feeds.
  Raising time limit on analytics cleaning.
  Handling another requests error when fetching icons.
This commit is contained in:
Samuel Clay 2015-11-30 09:19:04 -08:00
commit 952610e5e7
5 changed files with 23 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/reader/tasks.py
View file

@ -34,7 +34,7 @@ class FreshenHomepage(Task):
class CleanAnalytics(Task):
name = 'clean-analytics'
timeout = 720*10
hard = 720*10
def run(self, **kwargs):
logging.debug(" ---> Cleaning analytics... %s feed fetches" % (

2
apps/rss_feeds/icon_importer.py
View file

@ -202,6 +202,8 @@ class IconImporter(object):
requests.models.MissingSchema, requests.sessions.InvalidSchema,
requests.sessions.TooManyRedirects,
requests.models.InvalidURL,
requests.models.ChunkedEncodingError,
requests.models.ContentDecodingError,
LocationParseError, OpenSSLError, PyAsn1Error), e:
logging.debug(" ---> ~SN~FRFailed~FY to fetch ~FGfeed icon~FY: %s" % e)
if url:

7
apps/statistics/models.py
View file

@ -8,6 +8,7 @@ from apps.profile.models import Profile
from apps.statistics.rstats import RStats, round_time
from utils import json_functions as json
from utils import db_functions
from utils import log as logging
class MStatistics(mongo.Document):
key = mongo.StringField(unique=True)
@ -248,7 +249,11 @@ class MFeedback(mongo.Document):
@classmethod
def collect_feedback(cls):
data = urllib2.urlopen('https://getsatisfaction.com/newsblur/topics.widget').read()
try:
data = urllib2.urlopen('https://getsatisfaction.com/newsblur/topics.widget').read()
except (urllib2.HTTPError), e:
logging.debug(" ***> Failed to collect feedback: %s" % e)
return
data = json.decode(data[1:-1])
i = 0
if len(data):

2
settings.py
View file

@ -438,7 +438,7 @@ CELERYBEAT_SCHEDULE = {
'clean-analytics': {
'task': 'clean-analytics',
'schedule': datetime.timedelta(hours=12),
'options': {'queue': 'beat_tasks'},
'options': {'queue': 'beat_tasks', 'timeout': 720*10},
},
'clean-spam': {
'task': 'clean-spam',

16
utils/feed_fetcher.py
View file

@ -113,11 +113,17 @@ class FetchFeed:
etag=etag,
modified=modified)
except (TypeError, ValueError, KeyError, EOFError), e:
logging.debug(u' ***> [%-30s] ~FR%s, turning off headers.' %
logging.debug(u' ***> [%-30s] ~FRFeed fetch error: %s' %
(self.feed.title[:30], e))
pass
if not self.fpf:
try:
logging.debug(u' ***> [%-30s] ~FRTurning off headers...' %
(self.feed.title[:30]))
self.fpf = feedparser.parse(address, agent=USER_AGENT)
except (TypeError, ValueError, KeyError, EOFError), e:
logging.debug(u' ***> [%-30s] ~FR%s fetch failed: %s.' %
logging.debug(u' ***> [%-30s] ~FRFetch failed: %s.' %
(self.feed.title[:30], e))
return FEED_ERRHTTP, None
@ -522,7 +528,10 @@ class Dispatcher:
def refresh_feed(self, feed_id):
"""Update feed, since it may have changed"""
return Feed.objects.using('default').get(pk=feed_id)
try:
return Feed.objects.using('default').get(pk=feed_id)
except Feed.DoesNotExist:
return
def process_feed_wrapper(self, feed_queue):
delta = None
@ -643,6 +652,7 @@ class Dispatcher:
if not feed: continue
feed = self.refresh_feed(feed.pk)
if not feed: continue
if ((self.options['force']) or
(random.random() > .9) or