diff --git a/Makefile b/Makefile index eab53612a..645cb76e2 100644 --- a/Makefile +++ b/Makefile @@ -185,7 +185,7 @@ maintenance_off: # Provision firewall: - ansible-playbook ansible/all.yml -l db --tags firewall + ansible-playbook ansible/all.yml -l db --tags ufw oldfirewall: ANSIBLE_CONFIG=/srv/newsblur/ansible.old.cfg ansible-playbook ansible/all.yml -l db --tags firewall repairmongo: diff --git a/ansible/playbooks/setup_node.yml b/ansible/playbooks/setup_node.yml index cc2528fcd..78a66af60 100644 --- a/ansible/playbooks/setup_node.yml +++ b/ansible/playbooks/setup_node.yml @@ -1,7 +1,6 @@ --- - name: SETUP -> node containers hosts: node - become: true vars_files: - ../env_vars/base.yml vars: diff --git a/ansible/roles/base/tasks/main.yml b/ansible/roles/base/tasks/main.yml index 22638df1c..94c936e7c 100644 --- a/ansible/roles/base/tasks/main.yml +++ b/ansible/roles/base/tasks/main.yml @@ -6,6 +6,10 @@ tags: packages # ignore_errors: yes +- name: whoami + debug: + var: ansible_user_id + tags: whoami - name: Set timezone become: yes @@ -27,10 +31,20 @@ become: yes - name: Cloning oh-my-zsh - git: - repo: https://github.com/robbyrussell/oh-my-zsh - dest: /home/nb/.oh-my-zsh - force: yes + block: + - name: Cloning oh-my-zsh + git: + repo: https://github.com/robbyrussell/oh-my-zsh + dest: /home/nb/.oh-my-zsh + force: yes + rescue: + - name: chown oh-my-zsh + become: yes + file: + path: /home/nb/.oh-my-zsh + owner: nb + group: nb + recurse: yes - name: Copy toprc copy: src=toprc.txt dest=~/.toprc diff --git a/apps/rss_feeds/page_importer.py b/apps/rss_feeds/page_importer.py index ba6b66d68..e35c86cab 100644 --- a/apps/rss_feeds/page_importer.py +++ b/apps/rss_feeds/page_importer.py @@ -1,23 +1,28 @@ -import requests -import re -import traceback -import feedparser -import time -import urllib.request, urllib.error, urllib.parse import http.client +import re +import time +import traceback +import urllib.error +import urllib.parse +import urllib.request import zlib +from socket import error as SocketError + +import feedparser +import requests +from django.conf import settings from django.contrib.sites.models import Site from django.utils.encoding import smart_bytes -from mongoengine.queryset import NotUniqueError -from socket import error as SocketError -from django.conf import settings from django.utils.text import compress_string as compress_string_with_gzip -from utils import log as logging -from apps.rss_feeds.models import MFeedPage -from utils.feed_functions import timelimit, TimeoutError +from mongoengine.queryset import NotUniqueError from OpenSSL.SSL import Error as OpenSSLError from pyasn1.error import PyAsn1Error from sentry_sdk import capture_exception, flush + +from apps.rss_feeds.models import MFeedPage +from utils import log as logging +from utils.feed_functions import TimeoutError, timelimit + # from utils.feed_functions import mail_feed_error_to_admin BROKEN_PAGES = [ @@ -127,6 +132,7 @@ class PageImporter(object): return except (ValueError, urllib.error.URLError, http.client.BadStatusLine, http.client.InvalidURL, requests.exceptions.ConnectionError) as e: + logging.debug(' ***> [%-30s] Page fetch failed: %s' % (self.feed.log_title[:30], e)) self.feed.save_page_history(401, "Bad URL", e) try: fp = feedparser.parse(self.feed.feed_address) @@ -134,10 +140,8 @@ class PageImporter(object): return html feed_link = fp.feed.get('link', "") self.feed.save() - logging.debug(' ***> [%-30s] Page fetch failed: %s' % (self.feed.log_title[:30], e)) - except (urllib.error.HTTPError) as e: - self.feed.save_page_history(e.code, e.msg, e.fp.read()) except (http.client.IncompleteRead) as e: + logging.debug(' ***> [%-30s] Page fetch failed: %s' % (self.feed.log_title[:30], e)) self.feed.save_page_history(500, "IncompleteRead", e) except (requests.exceptions.RequestException, requests.packages.urllib3.exceptions.HTTPError) as e: diff --git a/hetzner migration.md b/hetzner migration.md index 5207bc853..d9eb089dd 100644 --- a/hetzner migration.md +++ b/hetzner migration.md @@ -46,3 +46,5 @@ ansible-playbook ansible/deploy.yml -l happ-web-01 --tags maintenance_off aps -l www -t haproxy # Change DNS to point to Hetzner make maintenance_off + +# Setup redis replicas (replica.conf port)