Merge branch 'master' into dejal

ansible/playbooks/setup_mongo.yml

@@ -6,18 +6,18 @@
     - motd_role: db
   vars_files:
     - ../env_vars/base.yml
-  
+
   roles:
-    - {role: 'base', tags: 'base'}
-    - {role: 'ufw', tags: 'ufw'}
-    - {role: 'docker', tags: 'docker'}
-    - {role: 'repo', tags: ['repo', 'pull']}
-    - {role: 'dnsmasq', tags: 'dnsmasq'}
-    - {role: 'consul', tags: 'consul'}
-    - {role: 'consul-client', tags: 'consul'}
-    - {role: 'mongo', tags: 'mongo'}
-    - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
-    - {role: 'mongo-exporter', tags: ['mongo-exporter', 'metrics']}
-    - {role: 'monitor', tags: 'monitor'}
-    - {role: 'flask_metrics', tags: ['flask-metrics', 'metrics']}
+    - { role: "base", tags: "base" }
+    - { role: "ufw", tags: "ufw" }
+    - { role: "docker", tags: "docker" }
+    - { role: "repo", tags: ["repo", "pull"] }
+    - { role: "dnsmasq", tags: "dnsmasq" }
+    - { role: "consul", tags: "consul" }
+    - { role: "consul-client", tags: "consul" }
+    - { role: "mongo", tags: "mongo" }
+    - { role: "node-exporter", tags: ["node-exporter", "metrics"] }
+    - { role: "mongo-exporter", tags: ["mongo-exporter", "metrics"] }
+    - { role: "monitor", tags: "monitor" }
+    - { role: "flask_metrics", tags: ["flask-metrics", "metrics"] }
     # - {role: 'benchmark', tags: 'benchmark'}

ansible/roles/mongo/tasks/main.yml

@@ -49,7 +49,7 @@
 
     - name: Create the mount point
       become: yes
-      file: 
+      file:
         path: "/mnt/{{ inventory_hostname | regex_replace('db-|-', '') }}"
         state: directory
         owner: "{{ ansible_effective_user_id|int }}"
@@ -64,7 +64,6 @@
         opts: defaults,discard
         state: mounted
 
-
     - name: Set permissions on mongo volume
       # become: yes
       file:
@@ -93,7 +92,6 @@
         force: yes
   when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-mongo-secondary', 'db-mongo-analytics']
 
-
 - name: Block for mongo volume on hetzner
   block:
     - name: Create backup directory
@@ -119,7 +117,7 @@
     # network_mode: default
     # networks:
     #   - name: newsblurnet
-    #     aliases: 
+    #     aliases:
     #       - mongo
     # ports:
     #   - "27017:27017"
@@ -148,7 +146,7 @@
         # network_mode: default
         # networks:
         #   - name: newsblurnet
-        #     aliases: 
+        #     aliases:
         #       - mongo
         # ports:
         #   - "27017:27017"
@@ -186,7 +184,7 @@
     network_mode: default
     networks:
       - name: newsblurnet
-        aliases: 
+        aliases:
           - mongo
     ports:
       - "27017:27017"
@@ -214,7 +212,7 @@
     network_mode: default
     networks:
       - name: newsblurnet
-        aliases: 
+        aliases:
           - mongo
     ports:
       - "27017:27017"
@@ -231,7 +229,7 @@
 - name: Create mongo database user
   shell:
     # Don't use this line below as it means there is already a username and password, so no need to set one
-    # sleep 2; docker exec mongo mongo -u "{{ mongodb_username }}" -p "{{ mongodb_password }}" --eval ' 
+    # sleep 2; docker exec mongo mongo -u "{{ mongodb_username }}" -p "{{ mongodb_password }}" --eval '
     cmd: >-
       sleep 2; docker exec mongo mongo --eval '
       db.createUser(
@@ -252,8 +250,9 @@
     - "'there are no users authenticated' not in auth_result.stdout"
   tags:
     - mongoauth
+    - never
 
-# - debug: 
+# - debug:
 #     msg: "{{ auth_result }}"
 #   tags:
 #     - mongoauth
@@ -281,9 +280,9 @@
 - name: Setup logrotate for mongo
   become: yes
   copy: src=logrotate.conf dest=/etc/logrotate.d/mongodb mode=0755
-  tags: 
+  tags:
     - logrotate
-  
+
 - name: Add sanity checkers cronjob for disk usage
   become: yes
   cron:
@@ -326,7 +325,6 @@
   tags:
     - mongo-backup
     - cron
-
 # - name: Add mongo starred_stories+stories backup
 #   cron:
 #     name: mongo starred/shared/all stories backup
@@ -338,7 +336,7 @@
 #     - mongo-backup
 
 # Renaming a db-mongo-primary3 to db-mongo-primary2:
-#  - Change hostname to db-mongo-primary2 on Digital Ocean 
+#  - Change hostname to db-mongo-primary2 on Digital Ocean
 #  -     make list; doctl compute droplet-action rename <id>  --droplet-name db-mongo-primary2
 #  - Change hostname to db-mongo-primary2 in /etc/hostname
 #  - make inventory
@@ -352,7 +350,7 @@
 #  - doctl compute droplet delete db-mongo3
 #  - tf state rm "digitalocean_droplet.db-mongo-primary-s[1]"
 #  - tf state rm "digitalocean_droplet.db-mongo-primary-s[2]"
-#  - tf state mv "digitalocean_droplet.db-mongo-primary-s[3]" "digitalocean_droplet.db-mongo-primary-s[1]" 
+#  - tf state mv "digitalocean_droplet.db-mongo-primary-s[3]" "digitalocean_droplet.db-mongo-primary-s[1]"
 #  - Change hostname to db-mongo2 in /etc/hostname
 #  - sudo hostname db-mongo-primary2
 

ansible/roles/ufw/templates/ufw_rules.sh.j2

@@ -41,3 +41,7 @@ apply_rule "route allow from {{ host }}" "FWD" "{{ host }}"
 apply_rule "allow from {{ host }}" "IN" "{{ host }}"
 apply_rule "route allow from {{ host }}" "FWD" "{{ host }}"
 {% endfor %}
+
+# Allow traffic on docker0 interface
+apply_rule "allow in on docker0" "IN" "docker0"
+apply_rule "allow out on docker0" "IN" "docker0"

apps/rss_feeds/management/commands/clean_txt_records.py

@@ -0,0 +1,56 @@
+import requests
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+class Command(BaseCommand):
+    help = 'Delete old TXT records for Let\'s Encrypt from DNSimple'
+
+    def handle(self, *args, **kwargs):
+        API_TOKEN = settings.DNSIMPLE_API_TOKEN
+        ACCOUNT_ID = settings.DNSIMPLE_ACCOUNT_ID
+        DOMAIN = "newsblur.com"
+        LETSECRYPT_PREFIX = '_acme-challenge'
+
+        headers = {
+            'Authorization': f'Bearer {API_TOKEN}',
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+        }
+
+        def get_txt_records():
+            records = []
+            page = 1
+            while True:
+                url = f'https://api.dnsimple.com/v2/{ACCOUNT_ID}/zones/{DOMAIN}/records?page={page}'
+                response = requests.get(url, headers=headers)
+                if response.status_code == 200:
+                    data = response.json().get('data', [])
+                    records.extend(data)
+                    if 'pagination' in response.json():
+                        pagination = response.json()['pagination']
+                        if pagination['current_page'] < pagination['total_pages']:
+                            page += 1
+                        else:
+                            break
+                    else:
+                        break
+                else:
+                    self.stderr.write(f"Failed to fetch records: {response.status_code} {response.text}")
+                    break
+            return records
+
+        def delete_record(record_id):
+            url = f'https://api.dnsimple.com/v2/{ACCOUNT_ID}/zones/{DOMAIN}/records/{record_id}'
+            response = requests.delete(url, headers=headers)
+            if response.status_code == 204:
+                self.stdout.write(f"Deleted record {record_id}")
+            else:
+                self.stderr.write(f"Failed to delete record {record_id}: {response.status_code} {response.text}")
+
+        records = get_txt_records()
+        self.stdout.write(f"Found {len(records)} records")
+        for record in records:
+            # self.stdout.write(f"Record: {record}")
+            if record['type'] == 'TXT' and record['name'].startswith(LETSECRYPT_PREFIX):
+                self.stdout.write(f"Deleting record {record['id']} {record['name']} {record['content']}")
+                delete_record(record['id'])

docker/haproxy/haproxy.consul.cfg.j2

@@ -213,7 +213,7 @@ backend postgres
     {% for host in groups.postgres %}
         server {{host}} {{host}}.node.nyc1.consul:5579
     {% endfor %}
-    server hdb-postgres-secondary hdb-redis-secondary.node.nyc1.consul:5579
+    # server hdb-postgres-secondary hdb-redis-secondary.node.nyc1.consul:5579
 
 backend mongo
     option httpchk GET /db_check/mongo

utils/monitor_newsletter_delivery.py

@@ -24,7 +24,7 @@ def main():
     delivered = stats["delivered"]["total"]
     accepted = stats["delivered"]["total"]
     bounced = stats["failed"]["permanent"]["total"] + stats["failed"]["temporary"]["total"]
-    if bounced / float(delivered) > 0.5:
+    if bounced / float(delivered) > 0.5 and bounced > 100:
         requests.post(
             "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME,
             auth=("api", settings.MAILGUN_ACCESS_KEY),