mirror of
https://github.com/viq/NewsBlur.git
synced 2025-09-18 21:43:31 +00:00
Nice number of feed fetchers
This commit is contained in:
Samuel Clay
parent
8974b0dd9a
commit
a4b7927b92
4 changed files with 66 additions and 55 deletions
|
|
@ -22,7 +22,7 @@ from apps.rss_feeds.models import MFeedPage, MFeedIcon
|
|||
from utils.facebook_fetcher import FacebookFetcher
|
||||
from utils import log as logging
|
||||
from utils.feed_functions import timelimit, TimeoutError
|
||||
from OpenSSL.SSL import Error as OpenSSLError
|
||||
from OpenSSL.SSL import Error as OpenSSLError, SESS_CACHE_NO_INTERNAL_STORE
|
||||
from pyasn1.error import PyAsn1Error
|
||||
from requests.packages.urllib3.exceptions import LocationParseError
|
||||
|
||||
|
|
@ -53,51 +53,51 @@ class IconImporter(object):
|
|||
image, image_file, icon_url = self.fetch_image_from_page_data()
|
||||
if not image:
|
||||
image, image_file, icon_url = self.fetch_image_from_path(force=self.force)
|
||||
|
||||
if image:
|
||||
image = self.normalize_image(image)
|
||||
try:
|
||||
color = self.determine_dominant_color_in_image(image)
|
||||
except (IndexError, ValueError, MemoryError):
|
||||
logging.debug(" ---> [%-30s] ~SN~FRFailed to measure icon" % self.feed.log_title[:30])
|
||||
return
|
||||
try:
|
||||
image_str = self.string_from_image(image)
|
||||
except TypeError:
|
||||
return
|
||||
|
||||
if len(image_str) > 500000:
|
||||
image = None
|
||||
if (image and
|
||||
(self.force or
|
||||
self.feed_icon.data != image_str or
|
||||
self.feed_icon.icon_url != icon_url or
|
||||
self.feed_icon.not_found or
|
||||
(settings.BACKED_BY_AWS.get('icons_on_s3') and not self.feed.s3_icon))):
|
||||
logging.debug(" ---> [%-30s] ~SN~FBIcon difference:~FY color:%s (%s/%s) data:%s url:%s notfound:%s no-s3:%s" % (
|
||||
self.feed.log_title[:30],
|
||||
self.feed_icon.color != color, self.feed_icon.color, color,
|
||||
self.feed_icon.data != image_str,
|
||||
self.feed_icon.icon_url != icon_url,
|
||||
self.feed_icon.not_found,
|
||||
settings.BACKED_BY_AWS.get('icons_on_s3') and not self.feed.s3_icon))
|
||||
self.feed_icon.data = image_str
|
||||
self.feed_icon.icon_url = icon_url
|
||||
self.feed_icon.color = color
|
||||
self.feed_icon.not_found = False
|
||||
self.feed_icon.save()
|
||||
if settings.BACKED_BY_AWS.get('icons_on_s3'):
|
||||
self.save_to_s3(image_str)
|
||||
if self.feed.favicon_color != color:
|
||||
self.feed.favicon_color = color
|
||||
self.feed.favicon_not_found = False
|
||||
self.feed.save(update_fields=['favicon_color', 'favicon_not_found'])
|
||||
|
||||
|
||||
if not image:
|
||||
self.feed_icon.not_found = True
|
||||
self.feed_icon.save()
|
||||
self.feed.favicon_not_found = True
|
||||
self.feed.save()
|
||||
return False
|
||||
|
||||
image = self.normalize_image(image)
|
||||
try:
|
||||
color = self.determine_dominant_color_in_image(image)
|
||||
except (IndexError, ValueError, MemoryError):
|
||||
logging.debug(" ---> [%-30s] ~SN~FRFailed to measure icon" % self.feed.log_title[:30])
|
||||
return
|
||||
try:
|
||||
image_str = self.string_from_image(image)
|
||||
except TypeError:
|
||||
return
|
||||
|
||||
if len(image_str) > 500000:
|
||||
image = None
|
||||
if (image and
|
||||
(self.force or
|
||||
self.feed_icon.data != image_str or
|
||||
self.feed_icon.icon_url != icon_url or
|
||||
self.feed_icon.not_found or
|
||||
(settings.BACKED_BY_AWS.get('icons_on_s3') and not self.feed.s3_icon))):
|
||||
logging.debug(" ---> [%-30s] ~SN~FBIcon difference:~FY color:%s (%s/%s) data:%s url:%s notfound:%s no-s3:%s" % (
|
||||
self.feed.log_title[:30],
|
||||
self.feed_icon.color != color, self.feed_icon.color, color,
|
||||
self.feed_icon.data != image_str,
|
||||
self.feed_icon.icon_url != icon_url,
|
||||
self.feed_icon.not_found,
|
||||
settings.BACKED_BY_AWS.get('icons_on_s3') and not self.feed.s3_icon))
|
||||
self.feed_icon.data = image_str
|
||||
self.feed_icon.icon_url = icon_url
|
||||
self.feed_icon.color = color
|
||||
self.feed_icon.not_found = False
|
||||
self.feed_icon.save()
|
||||
if settings.BACKED_BY_AWS.get('icons_on_s3'):
|
||||
self.save_to_s3(image_str)
|
||||
if self.feed.favicon_color != color:
|
||||
self.feed.favicon_color = color
|
||||
self.feed.favicon_not_found = False
|
||||
self.feed.save(update_fields=['favicon_color', 'favicon_not_found'])
|
||||
|
||||
return not self.feed.favicon_not_found
|
||||
|
||||
|
|
@ -382,8 +382,9 @@ class IconImporter(object):
|
|||
index_max = scipy.argmax(counts)
|
||||
peak = codes.astype(int)[index_max]
|
||||
color = "{:02x}{:02x}{:02x}".format(peak[0], peak[1], peak[2])
|
||||
color = self.feed.adjust_color(color[:6], 21)
|
||||
|
||||
return color[:6]
|
||||
return color
|
||||
|
||||
def string_from_image(self, image):
|
||||
output = BytesIO()
|
||||
|
|
|
|||
|
|
@ -910,20 +910,24 @@ class Feed(models.Model):
|
|||
self.feed_title,
|
||||
), end=' ')
|
||||
|
||||
def _split_favicon_color(self):
|
||||
color = self.favicon_color
|
||||
if color:
|
||||
splitter = lambda s, p: [s[i:i+p] for i in range(0, len(s), p)]
|
||||
red, green, blue = splitter(color[:6], 2)
|
||||
return red, green, blue
|
||||
return None, None, None
|
||||
def _split_favicon_color(self, color=None):
|
||||
if not color:
|
||||
color = self.favicon_color
|
||||
if not color:
|
||||
return None, None, None
|
||||
splitter = lambda s, p: [s[i:i+p] for i in range(0, len(s), p)]
|
||||
red, green, blue = splitter(color[:6], 2)
|
||||
return red, green, blue
|
||||
|
||||
def favicon_fade(self):
|
||||
red, green, blue = self._split_favicon_color()
|
||||
return self.adjust_color(adjust=30)
|
||||
|
||||
def adjust_color(self, color=None, adjust=0):
|
||||
red, green, blue = self._split_favicon_color(color=color)
|
||||
if red and green and blue:
|
||||
fade_red = hex(min(int(red, 16) + 35, 255))[2:].zfill(2)
|
||||
fade_green = hex(min(int(green, 16) + 35, 255))[2:].zfill(2)
|
||||
fade_blue = hex(min(int(blue, 16) + 35, 255))[2:].zfill(2)
|
||||
fade_red = hex(min(int(red, 16) + adjust, 255))[2:].zfill(2)
|
||||
fade_green = hex(min(int(green, 16) + adjust, 255))[2:].zfill(2)
|
||||
fade_blue = hex(min(int(blue, 16) + adjust, 255))[2:].zfill(2)
|
||||
return "%s%s%s" % (fade_red, fade_green, fade_blue)
|
||||
|
||||
def favicon_border(self):
|
||||
|
|
|
|||
|
|
@ -1771,7 +1771,7 @@ hr {
|
|||
}
|
||||
.NB-image-preview-small-left:not(.NB-story-layout-grid) .NB-view-river .NB-story-pane-west .NB-story-title,
|
||||
.NB-image-preview-large-left:not(.NB-story-layout-grid) .NB-view-river .NB-story-pane-west .NB-story-title {
|
||||
padding-left: 142px;
|
||||
padding-left: 120px;
|
||||
}
|
||||
|
||||
.NB-story-title .NB-storytitles-feed-border-inner,
|
||||
|
|
@ -1804,6 +1804,7 @@ hr {
|
|||
.NB-image-preview-small-left:not(.NB-story-layout-grid) .NB-story-title .NB-storytitles-sentiment,
|
||||
.NB-image-preview-large-left:not(.NB-story-layout-grid) .NB-story-title .NB-storytitles-sentiment {
|
||||
left: 90px;
|
||||
top: 24px;
|
||||
}
|
||||
.NB-story-title.NB-story-positive .NB-storytitles-sentiment {
|
||||
background: transparent url('/media/embed/icons/circular/g_icn_focus.png') no-repeat 13px 6px;
|
||||
|
|
@ -2068,6 +2069,9 @@ hr {
|
|||
overflow: hidden;
|
||||
width: 100px;
|
||||
line-height: 1.3em;
|
||||
word-break: break-all;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.NB-theme-feed-size-xs .NB-story-title .NB-story-feed .feed_title {
|
||||
font-size: 9px;
|
||||
|
|
@ -2109,6 +2113,7 @@ hr {
|
|||
.NB-image-preview-small-left:not(.NB-story-layout-grid) .NB-story-title .NB-story-manage-icon,
|
||||
.NB-image-preview-large-left:not(.NB-story-layout-grid) .NB-story-title .NB-story-manage-icon {
|
||||
left: 90px;
|
||||
top: 24px;
|
||||
}
|
||||
.NB-story-title:hover .NB-story-manage-icon {
|
||||
display: block;
|
||||
|
|
@ -2543,6 +2548,7 @@ hr {
|
|||
height: 16px;
|
||||
margin: 2px 0 0 0;
|
||||
line-height: 16px;
|
||||
word-break: break-all;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
.NB-story-grid .feed_favicon,
|
||||
|
|
|
|||
|
|
@ -391,7 +391,7 @@ resource "digitalocean_droplet" "db-mongo-analytics" {
|
|||
}
|
||||
|
||||
resource "digitalocean_droplet" "task-celery" {
|
||||
count = 40
|
||||
count = 69
|
||||
image = var.droplet_os
|
||||
name = "task-celery${count.index+1}"
|
||||
region = var.droplet_region
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue