NewsBlur/utils/json_fetcher.py

import datetime

import dateutil.parser
from django.conf import settings
from django.utils import feedgenerator

from utils import log as logging
from utils.json_functions import decode


class JSONFetcher:
    def __init__(self, feed, options=None):
        self.feed = feed
        self.options = options or {}

    def fetch(self, address, raw_feed):
        if not address:
            address = self.feed.feed_address

        json_feed = decode(raw_feed.content)
        if not json_feed:
            logging.debug("   ***> [%-30s] ~FRJSON fetch failed: %s" % (self.feed.log_title[:30], address))
            return

        data = {}
        data["title"] = json_feed.get("title", "[Untitled]")
        data["link"] = json_feed.get("home_page_url", "")
        data["description"] = json_feed.get("title", "")
        data["lastBuildDate"] = datetime.datetime.utcnow()
        data["generator"] = "NewsBlur JSON Feed - %s" % settings.NEWSBLUR_URL
        data["docs"] = None
        data["feed_url"] = json_feed.get("feed_url")

        rss = feedgenerator.Atom1Feed(**data)

        for item in json_feed.get("items", []):
            story_data = self.json_feed_story(item)
            rss.add_item(**story_data)

        return rss.writeString("utf-8")

    def json_feed_story(self, item):
        date_published = datetime.datetime.now()
        pubdate = item.get("date_published", None)
        if pubdate:
            date_published = dateutil.parser.parse(pubdate)
        authors = item.get("authors", item.get("author", {}))
        if isinstance(authors, list):
            author_name = ", ".join([author.get("name", "") for author in authors])
        else:
            author_name = authors.get("name", "")
        story = {
            "title": item.get("title", ""),
            "link": item.get("external_url", item.get("url", "")),
            "description": item.get("content_html", item.get("content_text", "")),
            "author_name": author_name,
            "categories": item.get("tags", []),
            "unique_id": str(item.get("id", item.get("url", ""))),
            "pubdate": date_published,
        }

        return story