Logging failed page fetch timeout.

Makefile

@@ -185,7 +185,7 @@ maintenance_off:
 
 # Provision
 firewall:
-	ansible-playbook ansible/all.yml -l db --tags firewall
+	ansible-playbook ansible/all.yml -l db --tags ufw
 oldfirewall:
 	ANSIBLE_CONFIG=/srv/newsblur/ansible.old.cfg ansible-playbook ansible/all.yml  -l db --tags firewall
 repairmongo:

ansible/playbooks/setup_node.yml

@@ -1,7 +1,6 @@
 ---
 - name: SETUP -> node containers
   hosts: node
-  become: true
   vars_files:
     - ../env_vars/base.yml
   vars:

ansible/roles/base/tasks/main.yml

@@ -6,6 +6,10 @@
   tags: packages
   # ignore_errors: yes
 
+- name: whoami
+  debug:
+    var: ansible_user_id
+  tags: whoami
 
 - name: Set timezone
   become: yes
@@ -27,10 +31,20 @@
   become: yes
   
 - name: Cloning oh-my-zsh
-  git:
+  block:
-    repo: https://github.com/robbyrussell/oh-my-zsh
+    - name: Cloning oh-my-zsh
-    dest: /home/nb/.oh-my-zsh
+      git:
-    force: yes
+        repo: https://github.com/robbyrussell/oh-my-zsh
+        dest: /home/nb/.oh-my-zsh
+        force: yes
+  rescue:
+    - name: chown oh-my-zsh
+      become: yes
+      file:
+        path: /home/nb/.oh-my-zsh
+        owner: nb
+        group: nb
+        recurse: yes
   
 - name: Copy toprc
   copy: src=toprc.txt dest=~/.toprc

apps/rss_feeds/page_importer.py

@@ -1,23 +1,28 @@
-import requests
-import re
-import traceback
-import feedparser
-import time
-import urllib.request, urllib.error, urllib.parse
 import http.client
+import re
+import time
+import traceback
+import urllib.error
+import urllib.parse
+import urllib.request
 import zlib
+from socket import error as SocketError
+
+import feedparser
+import requests
+from django.conf import settings
 from django.contrib.sites.models import Site
 from django.utils.encoding import smart_bytes
-from mongoengine.queryset import NotUniqueError
-from socket import error as SocketError
-from django.conf import settings
 from django.utils.text import compress_string as compress_string_with_gzip
-from utils import log as logging
+from mongoengine.queryset import NotUniqueError
-from apps.rss_feeds.models import MFeedPage
-from utils.feed_functions import timelimit, TimeoutError
 from OpenSSL.SSL import Error as OpenSSLError
 from pyasn1.error import PyAsn1Error
 from sentry_sdk import capture_exception, flush
+
+from apps.rss_feeds.models import MFeedPage
+from utils import log as logging
+from utils.feed_functions import TimeoutError, timelimit
+
 # from utils.feed_functions import mail_feed_error_to_admin
 
 BROKEN_PAGES = [
@@ -127,6 +132,7 @@ class PageImporter(object):
                 return
         except (ValueError, urllib.error.URLError, http.client.BadStatusLine, http.client.InvalidURL,
                 requests.exceptions.ConnectionError) as e:
+            logging.debug('   ***> [%-30s] Page fetch failed: %s' % (self.feed.log_title[:30], e))
             self.feed.save_page_history(401, "Bad URL", e)
             try:
                 fp = feedparser.parse(self.feed.feed_address)
@@ -134,10 +140,8 @@ class PageImporter(object):
                 return html
             feed_link = fp.feed.get('link', "")
             self.feed.save()
-            logging.debug('   ***> [%-30s] Page fetch failed: %s' % (self.feed.log_title[:30], e))
-        except (urllib.error.HTTPError) as e:
-            self.feed.save_page_history(e.code, e.msg, e.fp.read())
         except (http.client.IncompleteRead) as e:
+            logging.debug('   ***> [%-30s] Page fetch failed: %s' % (self.feed.log_title[:30], e))
             self.feed.save_page_history(500, "IncompleteRead", e)
         except (requests.exceptions.RequestException, 
                 requests.packages.urllib3.exceptions.HTTPError) as e:

hetzner migration.md

@@ -46,3 +46,5 @@ ansible-playbook ansible/deploy.yml -l happ-web-01 --tags maintenance_off
 aps -l www -t haproxy
 # Change DNS to point to Hetzner
 make maintenance_off
+
+# Setup redis replicas (replica.conf port)