Merge branch 'master' into dejal

ansible/playbooks/setup_mongo.yml

@@ -8,16 +8,16 @@
     - ../env_vars/base.yml
 
   roles:
-    - {role: 'base', tags: 'base'}
-    - {role: 'ufw', tags: 'ufw'}
-    - {role: 'docker', tags: 'docker'}
-    - {role: 'repo', tags: ['repo', 'pull']}
-    - {role: 'dnsmasq', tags: 'dnsmasq'}
-    - {role: 'consul', tags: 'consul'}
-    - {role: 'consul-client', tags: 'consul'}
-    - {role: 'mongo', tags: 'mongo'}
-    - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
-    - {role: 'mongo-exporter', tags: ['mongo-exporter', 'metrics']}
-    - {role: 'monitor', tags: 'monitor'}
-    - {role: 'flask_metrics', tags: ['flask-metrics', 'metrics']}
+    - { role: "base", tags: "base" }
+    - { role: "ufw", tags: "ufw" }
+    - { role: "docker", tags: "docker" }
+    - { role: "repo", tags: ["repo", "pull"] }
+    - { role: "dnsmasq", tags: "dnsmasq" }
+    - { role: "consul", tags: "consul" }
+    - { role: "consul-client", tags: "consul" }
+    - { role: "mongo", tags: "mongo" }
+    - { role: "node-exporter", tags: ["node-exporter", "metrics"] }
+    - { role: "mongo-exporter", tags: ["mongo-exporter", "metrics"] }
+    - { role: "monitor", tags: "monitor" }
+    - { role: "flask_metrics", tags: ["flask-metrics", "metrics"] }
     # - {role: 'benchmark', tags: 'benchmark'}

ansible/roles/mongo/tasks/main.yml

@@ -64,7 +64,6 @@
         opts: defaults,discard
         state: mounted
 
-
     - name: Set permissions on mongo volume
       # become: yes
       file:
@@ -93,7 +92,6 @@
         force: yes
   when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-mongo-secondary', 'db-mongo-analytics']
 
-
 - name: Block for mongo volume on hetzner
   block:
     - name: Create backup directory
@@ -252,6 +250,7 @@
     - "'there are no users authenticated' not in auth_result.stdout"
   tags:
     - mongoauth
+    - never
 
 # - debug:
 #     msg: "{{ auth_result }}"
@@ -326,7 +325,6 @@
   tags:
     - mongo-backup
     - cron
-
 # - name: Add mongo starred_stories+stories backup
 #   cron:
 #     name: mongo starred/shared/all stories backup

ansible/roles/ufw/templates/ufw_rules.sh.j2

@@ -41,3 +41,7 @@ apply_rule "route allow from {{ host }}" "FWD" "{{ host }}"
 apply_rule "allow from {{ host }}" "IN" "{{ host }}"
 apply_rule "route allow from {{ host }}" "FWD" "{{ host }}"
 {% endfor %}
+
+# Allow traffic on docker0 interface
+apply_rule "allow in on docker0" "IN" "docker0"
+apply_rule "allow out on docker0" "IN" "docker0"

apps/rss_feeds/management/commands/clean_txt_records.py

@@ -0,0 +1,56 @@
+import requests
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+class Command(BaseCommand):
+    help = 'Delete old TXT records for Let\'s Encrypt from DNSimple'
+
+    def handle(self, *args, **kwargs):
+        API_TOKEN = settings.DNSIMPLE_API_TOKEN
+        ACCOUNT_ID = settings.DNSIMPLE_ACCOUNT_ID
+        DOMAIN = "newsblur.com"
+        LETSECRYPT_PREFIX = '_acme-challenge'
+
+        headers = {
+            'Authorization': f'Bearer {API_TOKEN}',
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+        }
+
+        def get_txt_records():
+            records = []
+            page = 1
+            while True:
+                url = f'https://api.dnsimple.com/v2/{ACCOUNT_ID}/zones/{DOMAIN}/records?page={page}'
+                response = requests.get(url, headers=headers)
+                if response.status_code == 200:
+                    data = response.json().get('data', [])
+                    records.extend(data)
+                    if 'pagination' in response.json():
+                        pagination = response.json()['pagination']
+                        if pagination['current_page'] < pagination['total_pages']:
+                            page += 1
+                        else:
+                            break
+                    else:
+                        break
+                else:
+                    self.stderr.write(f"Failed to fetch records: {response.status_code} {response.text}")
+                    break
+            return records
+
+        def delete_record(record_id):
+            url = f'https://api.dnsimple.com/v2/{ACCOUNT_ID}/zones/{DOMAIN}/records/{record_id}'
+            response = requests.delete(url, headers=headers)
+            if response.status_code == 204:
+                self.stdout.write(f"Deleted record {record_id}")
+            else:
+                self.stderr.write(f"Failed to delete record {record_id}: {response.status_code} {response.text}")
+
+        records = get_txt_records()
+        self.stdout.write(f"Found {len(records)} records")
+        for record in records:
+            # self.stdout.write(f"Record: {record}")
+            if record['type'] == 'TXT' and record['name'].startswith(LETSECRYPT_PREFIX):
+                self.stdout.write(f"Deleting record {record['id']} {record['name']} {record['content']}")
+                delete_record(record['id'])

docker/haproxy/haproxy.consul.cfg.j2

@@ -213,7 +213,7 @@ backend postgres
     {% for host in groups.postgres %}
         server {{host}} {{host}}.node.nyc1.consul:5579
     {% endfor %}
-    server hdb-postgres-secondary hdb-redis-secondary.node.nyc1.consul:5579
+    # server hdb-postgres-secondary hdb-redis-secondary.node.nyc1.consul:5579
 
 backend mongo
     option httpchk GET /db_check/mongo

utils/monitor_newsletter_delivery.py

@@ -24,7 +24,7 @@ def main():
     delivered = stats["delivered"]["total"]
     accepted = stats["delivered"]["total"]
     bounced = stats["failed"]["permanent"]["total"] + stats["failed"]["temporary"]["total"]
-    if bounced / float(delivered) > 0.5:
+    if bounced / float(delivered) > 0.5 and bounced > 100:
         requests.post(
             "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME,
             auth=("api", settings.MAILGUN_ACCESS_KEY),