diff --git a/apps/rss_feeds/text_importer.py b/apps/rss_feeds/text_importer.py index e0676c302..4300a5ab9 100644 --- a/apps/rss_feeds/text_importer.py +++ b/apps/rss_feeds/text_importer.py @@ -57,6 +57,7 @@ class TextImporter: if not use_mercury or not results: logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY with Mercury, trying readability...", warn_color=False) + results = self.fetch_manually(skip_save=skip_save, return_document=return_document) return results @@ -106,10 +107,18 @@ class TextImporter: if not resp: return + @timelimit(5) + def extract_text(resp): + try: + text = resp.text + except (LookupError, TypeError): + text = resp.content + return text try: - text = resp.text - except (LookupError, TypeError): - text = resp.content + text = extract_text(resp) + except TimeoutError: + logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: timed out on resp.text") + return # if self.debug: # logging.user(self.request, "~FBOriginal text's website: %s" % text) @@ -227,6 +236,8 @@ class TextImporter: headers["content-type"] = "application/json" headers["x-api-key"] = mercury_api_key domain = Site.objects.get_current().domain + if settings.DOCKERBUILD: + domain = 'haproxy' url = f"https://{domain}/rss_feeds/original_text_fetcher?url={url}" try: diff --git a/newsblur_web/docker_local_settings.py b/newsblur_web/docker_local_settings.py index c2171a9c5..2214191ad 100644 --- a/newsblur_web/docker_local_settings.py +++ b/newsblur_web/docker_local_settings.py @@ -95,8 +95,7 @@ MONGO_DB = { } MONGO_ANALYTICS_DB = { 'name': 'nbanalytics', - 'host': 'db_mongo', - 'port': 29019, + 'host': 'db_mongo:29019', } MONGODB_SLAVE = { diff --git a/newsblur_web/settings.py b/newsblur_web/settings.py index eb73aac19..458af7a21 100644 --- a/newsblur_web/settings.py +++ b/newsblur_web/settings.py @@ -696,11 +696,13 @@ MONGO_ANALYTICS_DB_DEFAULTS = { 'name': 'nbanalytics', 'host': f'db_mongo_analytics:{MONGO_PORT}', 'alias': 'nbanalytics', - 'username': 'newsblur', - 'password': 'newsblur', } MONGO_ANALYTICS_DB = dict(MONGO_ANALYTICS_DB_DEFAULTS, **MONGO_ANALYTICS_DB) -MONGOANALYTICSDB = connect(db=MONGO_ANALYTICS_DB['name'], host=f"mongodb://{MONGO_ANALYTICS_DB['username']}:{MONGO_ANALYTICS_DB['password']}@{MONGO_ANALYTICS_DB['host']}/?authSource=admin", alias="nbanalytics") +if 'username' in MONGO_ANALYTICS_DB: + MONGOANALYTICSDB = connect(db=MONGO_ANALYTICS_DB['name'], host=f"mongodb://{MONGO_ANALYTICS_DB['username']}:{MONGO_ANALYTICS_DB['password']}@{MONGO_ANALYTICS_DB['host']}/?authSource=admin", alias="nbanalytics") +else: + MONGOANALYTICSDB = connect(db=MONGO_ANALYTICS_DB['name'], host=f"mongodb://{MONGO_ANALYTICS_DB['host']}/", alias="nbanalytics") + # ========= # = Redis = diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index 1efea9531..b33cd66c4 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -665,13 +665,18 @@ class FeedFetcherWorker: """Update feed, since it may have changed""" return Feed.get_by_id(feed_id) - def process_feed_wrapper(self, feed_queue): + def reset_database_connections(self): connection._connections = {} connection._connection_settings ={} connection._dbs = {} settings.MONGODB = connect(settings.MONGO_DB_NAME, **settings.MONGO_DB) - settings.MONGOANALYTICSDB = connect(db=settings.MONGO_ANALYTICS_DB['name'], host=f"mongodb://{settings.MONGO_ANALYTICS_DB['username']}:{settings.MONGO_ANALYTICS_DB['password']}@{settings.MONGO_ANALYTICS_DB['host']}/?authSource=admin", alias="nbanalytics") + if 'username' in settings.MONGO_ANALYTICS_DB: + settings.MONGOANALYTICSDB = connect(db=settings.MONGO_ANALYTICS_DB['name'], host=f"mongodb://{settings.MONGO_ANALYTICS_DB['username']}:{settings.MONGO_ANALYTICS_DB['password']}@{settings.MONGO_ANALYTICS_DB['host']}/?authSource=admin", alias="nbanalytics") + else: + settings.MONGOANALYTICSDB = connect(db=settings.MONGO_ANALYTICS_DB['name'], host=f"mongodb://{settings.MONGO_ANALYTICS_DB['host']}/", alias="nbanalytics") + def process_feed_wrapper(self, feed_queue): + self.reset_database_connections() delta = None current_process = multiprocessing.current_process()