public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-09-18 21:43:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'jammit' into social

Browse source 
* jammit:
  Adding new iphone mask image.
  Moving to proper unicode string for feed fetching.
  Colorizing feed fetching.
  Colorizing feed fetching.
  Adding two error classes to requests exception handling.
  Adding two error classes to requests exception handling.
  Adding two error classes to requests exception handling.
  jQuery 1.6.1 -> 1.7.1. Fixes massive Chrome issues.
  Adding shift+u as a shortcut for marking a story unread.
  Adding more advanced exception handling for new requests module, with fallback to urllib2.
  Adding urllib fallback to requests in page fetching.
  Adding a little bit more leeway in page importer by falling back to urllib2 when requests fail.
  Adding slight bit of color to feed fetcher.
  Revert "Revert "Switching to requests from urllib2/httplib. Bring on the page errors.""

Conflicts:
	assets.yml
	local_settings.py.template
	media/js/jquery-1.7.1.js
	media/js/jquery-1.7.js
	media/js/vendor/jquery-1.7.js
This commit is contained in:
Samuel Clay 2012-01-05 08:38:08 -08:00
commit 8f4dd6566c
17 changed files with 433 additions and 449 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
EGG-INFO/PKG-INFO
View file

@ -1,16 +0,0 @@
Metadata-Version: 1.0
Name: django-compress
Version: 1.0.1
Summary: django-compress provides an automated system for compressing CSS and JavaScript files
Home-page: http://code.google.com/p/django-compress/
Author: Andreas Pelme
Author-email: Andreas Pelme <andreas@pelme.se>
License: UNKNOWN
Description: UNKNOWN
Platform: UNKNOWN
Classifier: Environment :: Web Environment
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Topic :: Utilities

33
EGG-INFO/SOURCES.txt
View file

@ -1,33 +0,0 @@
README
setup.py
compress/__init__.py
compress/filter_base.py
compress/models.py
compress/signals.py
compress/utils.py
compress/conf/__init__.py
compress/conf/settings.py
compress/filters/__init__.py
compress/filters/csstidy/__init__.py
compress/filters/csstidy_python/__init__.py
compress/filters/csstidy_python/csstidy.py
compress/filters/csstidy_python/data.py
compress/filters/csstidy_python/optimizer.py
compress/filters/csstidy_python/output.py
compress/filters/csstidy_python/tools.py
compress/filters/jsmin/__init__.py
compress/filters/jsmin/jsmin.py
compress/filters/yui/__init__.py
compress/management/__init__.py
compress/management/commands/__init__.py
compress/management/commands/synccompress.py
compress/templatetags/__init__.py
compress/templatetags/compressed.py
compress/versioning/__init__.py
compress/versioning/base.py
compress/versioning/hash/__init__.py
compress/versioning/mtime/__init__.py
django_compress.egg-info/PKG-INFO
django_compress.egg-info/SOURCES.txt
django_compress.egg-info/dependency_links.txt
django_compress.egg-info/top_level.txt

1
EGG-INFO/dependency_links.txt
View file

@ -1 +0,0 @@

1
EGG-INFO/top_level.txt
View file

@ -1 +0,0 @@
compress

1
EGG-INFO/zip-safe
View file

@ -1 +0,0 @@

5
apps/rss_feeds/management/commands/refresh_feeds.py
View file

@ -5,6 +5,7 @@ from apps.rss_feeds.models import Feed
from optparse import make_option
from utils import feed_fetcher
from utils.management_functions import daemonize
import django
import socket
import datetime
import redis
@ -69,12 +70,14 @@ class Command(BaseCommand):
feeds_queue = []
for _ in range(num_workers):
feeds_queue.append([])
i = 0
for feed in feeds:
feeds_queue[i%num_workers].append(feed.pk)
i += 1
disp.add_jobs(feeds_queue, i)
django.db.connection.close()
print " ---> Fetching %s feeds..." % feeds.count()
disp.run_jobs()

14
apps/rss_feeds/models.py
View file

@ -142,7 +142,7 @@ class Feed(models.Model):
except IntegrityError, e:
duplicate_feed = Feed.objects.filter(feed_address=self.feed_address)
logging.debug("%s: %s" % (self.feed_address, duplicate_feed))
logging.debug(' ***> [%-30s] Feed deleted. Could not save: %s' % (self, e))
logging.debug(' ***> [%-30s] Feed deleted. Could not save: %s' % (unicode(self)[:30], e))
if duplicate_feed:
merge_feeds(self.pk, duplicate_feed[0].pk)
return duplicate_feed[0]
@ -277,7 +277,7 @@ class Feed(models.Model):
try:
feed_address = _1()
except TimeoutError:
logging.debug(' ---> [%-30s] Feed address check timed out...' % (unicode(self.feed_title)[:30]))
logging.debug(' ---> [%-30s] Feed address check timed out...' % (unicode(self)[:30]))
self.save_feed_history(505, 'Timeout', '')
feed_address = None
@ -769,19 +769,21 @@ class Feed(models.Model):
story_feed_id=self.pk,
).order_by('-story_date')
if stories.count() > trim_cutoff:
logging.debug(' ---> [%-30s] Found %s stories. Trimming to %s...' % (self, stories.count(), trim_cutoff))
logging.debug(' ---> [%-30s] ~FBFound %s stories. Trimming to ~SB%s~SN...' % (unicode(self)[:30], stories.count(), trim_cutoff))
try:
story_trim_date = stories[trim_cutoff].story_date
except IndexError, e:
logging.debug(' ***> [%-30s] Error trimming feed: %s' % (self, e))
logging.debug(' ***> [%-30s] ~BRError trimming feed: %s' % (unicode(self)[:30], e))
return
extra_stories = MStory.objects(story_feed_id=self.pk, story_date__lte=story_trim_date)
extra_stories_count = extra_stories.count()
extra_stories.delete()
print "Deleted %s stories, %s left." % (extra_stories_count, MStory.objects(story_feed_id=self.pk).count())
if verbose:
print "Deleted %s stories, %s left." % (extra_stories_count, MStory.objects(story_feed_id=self.pk).count())
userstories = MUserStory.objects(feed_id=self.pk, story_date__lte=story_trim_date)
if userstories.count():
print "Found %s user stories. Deleting..." % userstories.count()
if verbose:
print "Found %s user stories. Deleting..." % userstories.count()
userstories.delete()
def get_stories(self, offset=0, limit=25, force=False, slave=False):

72
apps/rss_feeds/page_importer.py
View file

@ -1,18 +1,16 @@
import urllib2, httplib
import requests
import re
import urlparse
import traceback
import feedparser
import time
import urllib2
import httplib
from django.conf import settings
from utils import log as logging
from apps.rss_feeds.models import MFeedPage
from utils.feed_functions import timelimit, mail_feed_error_to_admin
HEADERS = {
'User-Agent': 'NewsBlur Page Fetcher - http://www.newsblur.com',
'Connection': 'close',
}
BROKEN_PAGES = [
'tag:',
'info:',
@ -23,50 +21,76 @@ BROKEN_PAGES = [
class PageImporter(object):
def __init__(self, url, feed):
self.url = url
def __init__(self, feed):
self.feed = feed
@property
def headers(self):
s = requests.session()
s.config['keep_alive'] = False
return {
'User-Agent': 'NewsBlur Page Fetcher (%s subscriber%s) - %s '
'(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
'Safari/534.48.3)' % (
self.feed.num_subscribers,
's' if self.feed.num_subscribers != 1 else '',
settings.NEWSBLUR_URL
),
'Connection': 'close',
}
@timelimit(15)
def fetch_page(self):
if not self.url:
def fetch_page(self, urllib_fallback=False):
feed_link = self.feed.feed_link
if not feed_link:
self.save_no_page()
return
try:
if self.url.startswith('http'):
request = urllib2.Request(self.url, headers=HEADERS)
response = urllib2.urlopen(request)
time.sleep(0.01) # Grrr, GIL.
data = response.read()
elif any(self.url.startswith(s) for s in BROKEN_PAGES):
if feed_link.startswith('www'):
self.feed.feed_link = 'http://' + feed_link
if feed_link.startswith('http'):
if urllib_fallback:
request = urllib2.Request(feed_link, headers=self.headers)
response = urllib2.urlopen(request)
time.sleep(0.01) # Grrr, GIL.
data = response.read()
else:
response = requests.get(feed_link, headers=self.headers)
data = response.content
elif any(feed_link.startswith(s) for s in BROKEN_PAGES):
self.save_no_page()
return
else:
data = open(self.url, 'r').read()
data = open(feed_link, 'r').read()
html = self.rewrite_page(data)
self.save_page(html)
except (ValueError, urllib2.URLError, httplib.BadStatusLine, httplib.InvalidURL), e:
self.feed.save_page_history(401, "Bad URL", e)
fp = feedparser.parse(self.feed.feed_address)
self.feed.feed_link = fp.feed.get('link', "")
feed_link = fp.feed.get('link', "")
self.feed.save()
except (urllib2.HTTPError), e:
self.feed.save_page_history(e.code, e.msg, e.fp.read())
return
except (httplib.IncompleteRead), e:
self.feed.save_page_history(500, "IncompleteRead", e)
return
except (requests.exceptions.RequestException,
LookupError,
requests.packages.urllib3.exceptions.HTTPError), e:
logging.debug(' ***> [%-30s] Page fetch failed using requests: %s' % (self.feed, e))
return self.fetch_page(urllib_fallback=True)
except Exception, e:
logging.debug('[%d] ! -------------------------' % (self.feed.id,))
tb = traceback.format_exc()
logging.debug(tb)
logging.debug('[%d] ! -------------------------' % (self.feed.id,))
self.feed.save_page_history(500, "Error", tb)
mail_feed_error_to_admin(self.feed, e)
return
self.feed.save_page_history(200, "OK")
mail_feed_error_to_admin(self.feed, e, locals())
if not urllib_fallback:
self.fetch_page(urllib_fallback=True)
else:
self.feed.save_page_history(200, "OK")
def save_no_page(self):
self.feed.has_page = False

2
assets.yml
View file

@ -22,7 +22,7 @@ compress_assets: on
javascripts:
common:
- media/js/vendor/jquery-1.7.js
- media/js/vendor/jquery-1.7.1.js
- media/js/vendor/jquery.json.js
- media/js/vendor/jquery.easing.js
- media/js/vendor/jquery.newsblur.js

62
local_settings.py.template
View file

@ -1,27 +1,24 @@
import logging
import pymongo
DATABASES = {
'default': {
'NAME': 'newsblur',
'ENGINE': 'django.db.backends.mysql',
'USER': 'newsblur',
'PASSWORD': '',
'HOST': '127.0.0.1',
},
}
# ===================
# = Server Settings =
# ===================
MONGO_DB = {
'name': 'newsblur',
'host': '127.0.0.1',
'port': 27017,
'read_preference': pymongo.ReadPreference.PRIMARY,
}
ADMINS = (
('Samuel Clay', 'samuel@ofbrooklyn.com'),
)
SERVER_EMAIL = 'server@newsblur.com'
HELLO_EMAIL = 'hello@newsblur.com'
NEWSBLUR_URL = 'http://www.newsblur.com'
# ==================
# = Global Settngs =
# ==================
DEBUG = True
MEDIA_URL = '/media/'
SECRET_KEY = 'YOUR SECRET KEY'
CACHE_BACKEND = 'dummy:///'
@ -37,6 +34,34 @@ HOMEPAGE_USERNAME = 'conesus'
OAUTH_KEY = 'www.example.com'
OAUTH_SECRET = 'SECRET_KEY_FROM_GOOGLE'
S3_ACCESS_KEY = 'XXX'
S3_SECRET = 'SECRET'
S3_BACKUP_BUCKET = 'newsblur_backups'
# =============
# = Databases =
# =============
DATABASES = {
'default': {
'NAME': 'newsblur',
'ENGINE': 'django.db.backends.mysql',
'USER': 'newsblur',
'PASSWORD': '',
'HOST': '127.0.0.1',
},
}
MONGO_DB = {
'name': 'newsblur',
'host': '127.0.0.1',
'port': 27017
}
MONGODB_SLAVE = {
'host': '127.0.0.1'
}
# Celery RabbitMQ Broker
BROKER_HOST = "127.0.0.1"
@ -65,6 +90,7 @@ if len(logging._handlerList) < 1:
format='%(asctime)-12s: %(message)s',
datefmt='%b %d %H:%M:%S',
handler=logging.StreamHandler)
<<<<<<< HEAD
S3_ACCESS_KEY = 'XXX'
S3_SECRET = 'SECRET'
@ -78,3 +104,5 @@ FACEBOOK_APP_ID = '111111111111111'
FACEBOOK_SECRET = '99999999999999999999999999999999'
TWITTER_CONSUMER_KEY = 'ooooooooooooooooooooo'
TWITTER_CONSUMER_SECRET = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
=======
>>>>>>> jammit

BIN
media/img/mobile/iphone_512_mask.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 40 KiB

7
media/js/newsblur/reader/reader.js
View file

@ -7080,6 +7080,13 @@
e.preventDefault();
self.open_river_stories();
});
$document.bind('keydown', 'shift+u', function(e) {
e.preventDefault();
if (self.active_story) {
var story_id = self.active_story.id;
self.mark_story_as_unread(story_id);
}
});
}
};

608
media/js/vendor/jquery-1.7.js → media/js/vendor/jquery-1.7.1.js vendored
View file

File diff suppressed because it is too large Load diff

22
settings.py
View file

@ -5,6 +5,18 @@ from mongoengine import connect
import redis
from utils import jammit
# ===================
# = Server Settings =
# ===================
ADMINS = (
('Samuel Clay', 'samuel@ofbrooklyn.com'),
)
SERVER_EMAIL = 'server@newsblur.com'
HELLO_EMAIL = 'hello@newsblur.com'
NEWSBLUR_URL = 'http://www.newsblur.com'
# ===========================
# = Directory Declaractions =
# ===========================
@ -27,14 +39,10 @@ if '/utils' not in ' '.join(sys.path):
sys.path.append(UTILS_ROOT)
if '/vendor' not in ' '.join(sys.path):
sys.path.append(VENDOR_ROOT)
# ===================
# = Global Settings =
# ===================
ADMINS = (
('Samuel Clay', 'samuel@ofbrooklyn.com'),
)
TEST_DEBUG = False
SEND_BROKEN_LINK_EMAILS = False
MANAGERS = ADMINS
@ -159,7 +167,7 @@ LOGGING = {
COMPRESS_JS = {
'all': {
'source_filenames': (
'js/jquery-1.6.1.js',
'js/jquery-1.7.1.js',
'js/inflector.js',
'js/jquery.json.js',
'js/jquery.easing.js',
@ -216,7 +224,7 @@ COMPRESS_JS = {
},
'mobile': {
'source_filenames': (
'js/jquery-1.6.1.js',
'js/jquery-1.7.1.js',
'js/mobile/jquery.mobile-1.0b1.js',
'js/jquery.ajaxmanager.3.js',
'js/underscore.js',
@ -310,8 +318,6 @@ SESSION_ENGINE = "django.contrib.sessions.backends.db"
TEST_RUNNER = "utils.testrunner.TestRunner"
SESSION_COOKIE_NAME = 'newsblur_sessionid'
SESSION_COOKIE_AGE = 60*60*24*365*2 # 2 years
SERVER_EMAIL = 'server@newsblur.com'
HELLO_EMAIL = 'hello@newsblur.com'
# ===========
# = Logging =

29
utils/feed_fetcher.py
View file

@ -23,7 +23,6 @@ import redis
# Refresh feed code adapted from Feedjack.
# http://feedjack.googlecode.com
URL = 'http://www.newsblur.com/'
SLOWFEED_WARNING = 10
ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
FEED_OK, FEED_SAME, FEED_ERRPARSE, FEED_ERRHTTP, FEED_ERREXC = range(5)
@ -46,7 +45,7 @@ class FetchFeed:
Uses feedparser to download the feed. Will be parsed later.
"""
identity = self.get_identity()
log_msg = u'%2s ---> [%-30s] Fetching feed (%d), last update: %s' % (identity,
log_msg = u'%2s ---> [%-30s] ~FYFetching feed (~FB%d~FY), last update: %s' % (identity,
unicode(self.feed)[:30],
self.feed.id,
datetime.datetime.now() - self.feed.last_update)
@ -63,7 +62,7 @@ class FetchFeed:
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3 (NewsBlur Feed Fetcher - %s subscriber%s - %s)' % (
self.feed.num_subscribers,
's' if self.feed.num_subscribers != 1 else '',
URL
settings.NEWSBLUR_URL
)
self.fpf = feedparser.parse(self.feed.feed_address,
@ -288,7 +287,7 @@ class Dispatcher:
ENTRY_SAME: 0,
ENTRY_ERR: 0
}
start_time = datetime.datetime.utcnow()
start_time = time.time()
ret_feed = FEED_ERREXC
try:
feed = self.refresh_feed(feed_id)
@ -345,12 +344,12 @@ class Dispatcher:
(ret_feed == FEED_OK or
(ret_feed == FEED_SAME and feed.stories_last_month > 10)))):
logging.debug(u' ---> [%-30s] Fetching page: %s' % (unicode(feed)[:30], feed.feed_link))
page_importer = PageImporter(feed.feed_link, feed)
logging.debug(u' ---> [%-30s] ~FYFetching page: %s' % (unicode(feed)[:30], feed.feed_link))
page_importer = PageImporter(feed)
try:
page_importer.fetch_page()
except TimeoutError, e:
logging.debug(' ---> [%-30s] Page fetch timed out...' % (unicode(feed)[:30]))
logging.debug(' ---> [%-30s] ~FRPage fetch timed out...' % (unicode(feed)[:30]))
feed.save_page_history(555, 'Timeout', '')
except Exception, e:
logging.debug('[%d] ! -------------------------' % (feed_id,))
@ -361,12 +360,12 @@ class Dispatcher:
fetched_feed = None
mail_feed_error_to_admin(feed, e)
logging.debug(u' ---> [%-30s] Fetching icon: %s' % (unicode(feed)[:30], feed.feed_link))
logging.debug(u' ---> [%-30s] ~FYFetching icon: %s' % (unicode(feed)[:30], feed.feed_link))
icon_importer = IconImporter(feed, force=self.options['force'])
try:
icon_importer.save()
except TimeoutError, e:
logging.debug(' ---> [%-30s] Icon fetch timed out...' % (unicode(feed)[:30]))
logging.debug(' ---> [%-30s] ~FRIcon fetch timed out...' % (unicode(feed)[:30]))
feed.save_page_history(556, 'Timeout', '')
except Exception, e:
logging.debug('[%d] ! -------------------------' % (feed_id,))
@ -376,23 +375,23 @@ class Dispatcher:
# feed.save_feed_history(560, "Icon Error", tb)
mail_feed_error_to_admin(feed, e)
else:
logging.debug(u' ---> [%-30s] Skipping page fetch: %s (%s on %s stories) %s' % (unicode(feed)[:30], unicode(feed.feed_link)[:30], self.feed_trans[ret_feed], feed.stories_last_month, '' if feed.has_page else ' [HAS NO PAGE]'))
logging.debug(u' ---> [%-30s] ~FBSkipping page fetch: (%s on %s stories) %s' % (unicode(feed)[:30], self.feed_trans[ret_feed], feed.stories_last_month, '' if feed.has_page else ' [HAS NO PAGE]'))
feed = self.refresh_feed(feed_id)
delta = datetime.datetime.utcnow() - start_time
delta = time.time() - start_time
feed.last_load_time = max(1, delta.seconds)
feed.last_load_time = round(delta)
feed.fetched_once = True
try:
feed.save()
except IntegrityError:
logging.debug(" ---> [%-30s] IntegrityError on feed: %s" % (unicode(feed)[:30], feed.feed_address,))
logging.debug(" ---> [%-30s] ~FRIntegrityError on feed: %s" % (unicode(feed)[:30], feed.feed_address,))
if ret_entries[ENTRY_NEW]:
self.publish_to_subscribers(feed)
done_msg = (u'%2s ---> [%-30s] Processed in %s (%s) [%s]' % (
identity, feed.feed_title[:30], unicode(delta),
done_msg = (u'%2s ---> [%-30s] ~FYProcessed in ~FG~SB%.4ss~FY~SN (~FB%s~FY) [%s]' % (
identity, feed.feed_title[:30], delta,
feed.pk, self.feed_trans[ret_feed],))
logging.debug(done_msg)

7
utils/feed_functions.py
View file

@ -176,13 +176,14 @@ def add_object_to_folder(obj, folder, folders, parent='', added=False):
folders[k][f_k] = add_object_to_folder(obj, folder, f_v, f_k, added)
return folders
def mail_feed_error_to_admin(feed, e):
def mail_feed_error_to_admin(feed, e, local_vars=None):
# Mail the admins with the error
exc_info = sys.exc_info()
subject = 'Feed update error: %s' % repr(e)
message = 'Traceback:\n%s\n\Feed:\n%s' % (
message = 'Traceback:\n%s\n\Feed:\n%s\nLocals:\n%s' % (
'\n'.join(traceback.format_exception(*exc_info)),
pprint.pformat(feed.__dict__)
pprint.pformat(feed.__dict__),
pprint.pformat(local_vars)
)
# print message
mail_admins(subject, message)

2
utils/log.py
View file

@ -37,7 +37,7 @@ def user(u, msg):
elif 'WP7' in user_agent:
platform = 'WP7'
premium = '*' if u.is_authenticated() and u.profile.is_premium else ''
username = cipher(u.__unicode__()) if settings.CIPHER_USERNAMES else u
username = cipher(unicode(u)) if settings.CIPHER_USERNAMES else u
info(' ---> [~FB~SN%-6s~SB] [%s%s] %s' % (platform, username, premium, msg))
def cipher(msg):