public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-08-31 21:41:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixing assortment of small bugs.

Browse source
This commit is contained in:
Samuel Clay 2013-01-28 15:43:00 -08:00
parent 8fbc9f5174
commit 7e52fc37ef
4 changed files with 40 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
apps/analyzer/views.py
View file

@ -3,7 +3,7 @@ from utils import log as logging
from django.shortcuts import get_object_or_404
from django.views.decorators.http import require_POST
from django.conf import settings
# from mongoengine.queryset import OperationError
from mongoengine.queryset import NotUniqueError
from apps.rss_feeds.models import Feed
from apps.reader.models import UserSubscription
from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag
@ -80,10 +80,10 @@ def save_classifier(request):
if content_type == 'feed':
if not post_content.startswith('social:'):
classifier_dict['feed_id'] = post_content
# try:
classifier, created = ClassifierCls.objects.get_or_create(**classifier_dict)
# except OperationError:
# continue
try:
classifier, created = ClassifierCls.objects.get_or_create(**classifier_dict)
except NotUniqueError:
continue
if score == 0:
classifier.delete()
elif classifier.score != score:

15
apps/rss_feeds/models.py
View file

@ -1573,6 +1573,7 @@ class MStarredStory(mongo.Document):
story_content_z = mongo.BinaryField()
story_original_content = mongo.StringField()
story_original_content_z = mongo.BinaryField()
original_text_z = mongo.BinaryField()
story_content_type = mongo.StringField(max_length=255)
story_author_name = mongo.StringField()
story_permalink = mongo.StringField()
@ -1611,7 +1612,19 @@ class MStarredStory(mongo.Document):
@property
def guid_hash(self):
return hashlib.sha1(self.story_guid).hexdigest()[:6]
def fetch_original_text(self, force=False, request=None):
original_text_z = self.original_text_z
if not original_text_z or force:
ti = TextImporter(self, request=request)
original_text = ti.fetch()
else:
logging.user(request, "~FYFetching ~FGoriginal~FY story text, ~SBfound.")
original_text = zlib.decompress(original_text_z)
return original_text
class MFeedFetchHistory(mongo.Document):
feed_id = mongo.IntField()

17
apps/rss_feeds/text_importer.py
View file

@ -23,13 +23,18 @@ class TextImporter:
'Connection': 'close',
}
def fetch(self):
html = requests.get(self.story.story_permalink, headers=self.headers)
original_text_doc = readability.Document(html.text, url=html.url, debug=settings.DEBUG)
content = original_text_doc.summary(html_partial=True)
def fetch(self, skip_save=False):
try:
html = requests.get(self.story.story_permalink, headers=self.headers)
original_text_doc = readability.Document(html.text, url=html.url, debug=settings.DEBUG)
content = original_text_doc.summary(html_partial=True)
except:
content = None
if content:
self.story.original_text_z = zlib.compress(content)
self.story.save()
if not skip_save:
self.story.original_text_z = zlib.compress(content)
self.story.save()
logging.user(self.request, "~SN~FYFetched ~FGoriginal text~FY: now ~SB%s bytes~SN vs. was ~SB%s bytes" % (
len(unicode(content)),
self.story.story_content_z and len(zlib.decompress(self.story.story_content_z))

14
apps/social/models.py
View file

@ -1899,11 +1899,17 @@ class MSharedStory(mongo.Document):
return image_sizes
def fetch_original_text(self):
ti = TextImporter(self)
original_text_doc = ti.fetch()
def fetch_original_text(self, force=False, request=None):
original_text_z = self.original_text_z
return original_text_doc
if not original_text_z or force:
ti = TextImporter(self, request=request)
original_text = ti.fetch()
else:
logging.user(request, "~FYFetching ~FGoriginal~FY story text, ~SBfound.")
original_text = zlib.decompress(original_text_z)
return original_text
class MSocialServices(mongo.Document):
user_id = mongo.IntField()