Updating consul and docker for arm64. Using old redis ports.

ansible/playbooks/setup_mongo.yml

@@ -8,13 +8,13 @@
     - ../env_vars/base.yml
   
   roles:
-    # - {role: 'base', tags: 'base'}
-    # - {role: 'ufw', tags: 'ufw'}
-    # - {role: 'docker', tags: 'docker'}
-    # - {role: 'repo', tags: ['repo', 'pull']}
-    # - {role: 'dnsmasq', tags: 'dnsmasq'}
-    # - {role: 'consul', tags: 'consul'}
-    # - {role: 'consul-client', tags: 'consul'}
+    - {role: 'base', tags: 'base'}
+    - {role: 'ufw', tags: 'ufw'}
+    - {role: 'docker', tags: 'docker'}
+    - {role: 'repo', tags: ['repo', 'pull']}
+    - {role: 'dnsmasq', tags: 'dnsmasq'}
+    - {role: 'consul', tags: 'consul'}
+    - {role: 'consul-client', tags: 'consul'}
     - {role: 'mongo', tags: 'mongo'}
     - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
     - {role: 'mongo-exporter', tags: ['mongo-exporter', 'metrics']}

ansible/playbooks/setup_task.yml

@@ -7,13 +7,13 @@
     - update_apt_cache: yes
     - motd_role: task
   roles:
-    - {role: 'base', tags: 'base'}
-    # - {role: 'ufw', tags: 'ufw'}
-    - {role: 'docker', tags: 'docker'}
-    - {role: 'repo', tags: ['repo', 'pull']}
-    - {role: 'dnsmasq', tags: 'dnsmasq'}
+    # - {role: 'base', tags: 'base'}
+    # - {role: 'docker', tags: 'docker'}
+    # - {role: 'repo', tags: ['repo', 'pull']}
+    # - {role: 'dnsmasq', tags: 'dnsmasq'}
     - {role: 'consul', tags: 'consul'}
     - {role: 'consul-client', tags: 'consul'}
     - {role: 'apns', tags: 'apns'}
     - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
     - {role: 'celery_task', tags: 'celery'}
+    - {role: 'ufw', tags: 'ufw'}

ansible/roles/consul-client/templates/consul_client.json.j2

@@ -9,7 +9,7 @@
   "log_file": "/var/log/consul/consul.log",
   "enable_syslog": true,
   "retry_join": [{{ consul_manager_ip.stdout|trim }}],
-  {% if inventory_hostname.startswith("hdb") %}
+  {% if inventory_hostname.startswith("hdb") and inventory_hostname|regex_replace("\-?\d+", "") not in ["hdb-mongo-analytics", "hdb-sentry"] %}
   "advertise_addr": "{% raw %}{{ GetAllInterfaces | include \"name\" \"^enp\" | include \"flags\" \"forwardable|up\" | attr \"address\" }}{% endraw %}",
   {% else %}
   "advertise_addr": "{% raw %}{{ GetAllInterfaces | include \"name\" \"^eth\" | include \"flags\" \"forwardable|up\" | attr \"address\" }}{% endraw %}",

ansible/roles/consul/tasks/main.yml

@@ -12,11 +12,21 @@
   apt_repository:
     repo: "deb [arch=amd64] https://apt.releases.hashicorp.com {{ ansible_distribution_release }} main"
 
+- name: Add the official HashiCorp Linux repository.
+  become: yes
+  apt_repository:
+    repo: "deb [arch=arm64] https://apt.releases.hashicorp.com {{ ansible_distribution_release }} main"
+
+- name: Update apt
+  become: yes
+  apt:
+    update_cache: yes
+
 - name: Installing Consul
   become: yes
   apt:
     allow_downgrades: yes
-    pkg: consul=1.10.4
+    pkg: consul=1.10.12-1
     state: present
 
 - name: Register Manager IP

ansible/roles/docker/tasks/install-Ubuntu.yml

@@ -15,25 +15,37 @@
     state: present
   with_items: "{{ docker_prerequisite_packages_Ubuntu }}"
 
-- name: Install prerequisite packages (for Ubuntu 14.04 only)
-  apt:
-    name: "{{ item.package }}"
-    state: present
-  with_items: "{{ docker_prerequisite_packages_Ubuntu_1404 }}"
-  when: ansible_distribution_version == '14.04'
-
-- name: Import Docker CE repository gpg key
+- name: Download Docker GPG key
   become: yes
-  apt_key:
-    url: https://download.docker.com/linux/ubuntu/gpg
-    state: present
-    id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
-
-- name: Add Docker CE repository
+  shell:
+    cmd: "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg"
+    creates: /usr/share/keyrings/docker-archive-keyring.gpg
+    
+- name: Set up the Docker repository with the correct GPG key
   become: yes
-  apt_repository:
-    repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
+  ansible.builtin.apt_repository:
+    repo: "deb [arch={{ ansible_architecture }} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
     state: present
+    update_cache: yes
+
+- name: Set up the Docker repository with the correct GPG key
+  become: yes
+  ansible.builtin.apt_repository:
+    repo: "deb [arch=arm64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
+    state: present
+    update_cache: yes
+
+- name: Set up the Docker repository with the correct GPG key
+  become: yes
+  ansible.builtin.apt_repository:
+    repo: "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
+    state: present
+    update_cache: yes
+
+- name: Update APT cache
+  become: yes
+  ansible.builtin.apt:
+    update_cache: yes
 
 - name: Install Docker CE
   become: yes

ansible/roles/mongo/tasks/main.yml

@@ -1,4 +1,11 @@
 ---
+- name: Set mongo_analytics_secondary
+  set_fact:
+    mongo_analytics_secondary: yes
+  when: (inventory_hostname | regex_replace('\-?[0-9]+', '')) not in ['db-mongo-analytics']
+  tags:
+    - consul
+
 - name: Permissions for mongo
   become: yes
   file:
@@ -259,7 +266,7 @@
   template:
     src: consul_service.analytics.json
     dest: /etc/consul.d/mongo.json
-  when: (inventory_hostname | regex_replace('[0-9]+', '')) == 'db-mongo-analytics'
+  when: (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ['db-mongo-analytics', 'hdb-mongo-analytics']
   notify:
     - reload consul
 

ansible/roles/mongo/templates/consul_service.analytics.json

@@ -1,6 +1,10 @@
 {
     "service": {
-        "name": "db-mongo-analytics",
+        {% if not mongo_analytics_secondary %}
+            "name": "db-mongo-analytics",
+        {% else %}
+            "name": "db-mongo-analytics-secondary",
+        {% endif %}
         "id": "{{ inventory_hostname }}",
         "tags": [
             "db"

newsblur_web/settings.py

@@ -1,6 +1,7 @@
-import sys
 import os
-import yaml 
+import sys
+
+import yaml
 
 # ===========================
 # = Directory Declaractions =
@@ -25,20 +26,22 @@ if '/utils' not in ' '.join(sys.path):
 if '/vendor' not in ' '.join(sys.path):
     sys.path.append(VENDOR_ROOT)
 
-import logging
 import datetime
+import logging
+import re
+
+import boto3
+import django.http
+import paypalrestsdk
 import redis
 import sentry_sdk
-import paypalrestsdk
-from sentry_sdk.integrations.django import DjangoIntegration
-from sentry_sdk.integrations.redis import RedisIntegration
-from sentry_sdk.integrations.celery import CeleryIntegration
-import django.http
-import re
 from mongoengine import connect
 from pymongo import monitoring
+from sentry_sdk.integrations.celery import CeleryIntegration
+from sentry_sdk.integrations.django import DjangoIntegration
+from sentry_sdk.integrations.redis import RedisIntegration
+
 from utils.mongo_command_monitor import MongoCommandLogger
-import boto3
 
 # ===================
 # = Server Settings =
@@ -762,11 +765,16 @@ if DOCKERBUILD:
     REDIS_SESSION_PORT = 6579
     REDIS_PUBSUB_PORT = 6579
 else:
-    # REDIS_PORT = 6379
+    REDIS_PORT = 6379
     REDIS_STORY_PORT = 6380
     REDIS_USER_PORT = 6381
     REDIS_SESSION_PORT = 6382
     REDIS_PUBSUB_PORT = 6383
+    # Until redis moves to hetzner, use old ports
+    REDIS_STORY_PORT = REDIS_PORT
+    REDIS_USER_PORT = REDIS_PORT
+    REDIS_SESSION_PORT = REDIS_PORT
+    REDIS_PUBSUB_PORT = REDIS_PORT
 
 if REDIS_USER is None:
     # REDIS has been renamed to REDIS_USER. 
@@ -918,6 +926,7 @@ django.http.request.host_validation_re = re.compile(r"^([a-z0-9.-_\-]+|\[[a-f0-9
 
 from django.contrib import auth
 
+
 def monkey_patched_get_user(request):
     """
     Return the user model instance associated with the given request session.