Secondary redis is now port separated.

ansible/inventories/hetzner.yml

@@ -3,8 +3,11 @@ strict: False
 
 groups:
 
+  hall: inventory_hostname.startswith('h')
+  
   haproxy: inventory_hostname.startswith('hwww')
 
+  happ: inventory_hostname.startswith('happ')
   web: inventory_hostname.startswith('happ')
   app: inventory_hostname.startswith('happ')
   django: inventory_hostname.startswith('happ-web')
@@ -13,6 +16,7 @@ groups:
   push: inventory_hostname.startswith('happ-push')
   blogs: inventory_hostname.startswith('blog')
 
+  hnode: inventory_hostname.startswith('hnode')
   node: inventory_hostname.startswith('hnode')
   node_socket: inventory_hostname.startswith('hnode-socket')
   node_images: inventory_hostname.startswith('hnode-images')
@@ -22,12 +26,14 @@ groups:
 
   # debugs: inventory_hostname.startswith('hdebug')
 
+  htask: inventory_hostname.startswith('htask')
   task: inventory_hostname.startswith('htask')
   celery: inventory_hostname.startswith('htask-celery')
   work: inventory_hostname.startswith('htask-work')
   
   staging: inventory_hostname.startswith('hstaging')
   
+  hdb: inventory_hostname.startswith('hdb')
   db: inventory_hostname.startswith('hdb')
   search: inventory_hostname.startswith('hdb-elasticsearch')
   elasticsearch: inventory_hostname.startswith('hdb-elasticsearch')

ansible/roles/base/tasks/main.yml

@@ -6,6 +6,13 @@
   tags: packages
   # ignore_errors: yes
 
+
+- name: Set timezone
+  become: yes
+  ansible.builtin.timezone:
+    name: 'America/New_York'
+  tags: timezone
+
 - name: Copy zshrc
   template: 
     src: zshrc.txt.j2

ansible/roles/redis/tasks/main.yml

@@ -1,4 +1,19 @@
 ---
+- name: Extract part of hostname to determine container name
+  set_fact:
+    redis_role: "{{ inventory_hostname.split('-')[2] }}"
+    redis_port: 6379
+    redis_secondary_ports:
+      story: 6380
+      user: 6381
+      session: 6382
+      pubsub: 6383
+
+- name: Set redis_port for secondary redis servers
+  set_fact:
+    redis_port: "{{ redis_secondary_ports[redis_role] }}"
+  when: inventory_hostname in ['hdb-redis-story-2', 'hdb-redis-user-2', 'hdb-redis-session-2', 'hdb-redis-pubsub-2']
+
 - name: Install sysfsutils for disabling transparent huge pages
   become: yes
   package:
@@ -31,7 +46,7 @@
 - name: Template redis_replica.conf file
   template:
     src: /srv/newsblur/docker/redis/redis_replica.conf.j2
-    dest: /srv/newsblur/docker/redis/redis_replica.conf
+    dest: "/srv/newsblur/docker/redis/redis_{{ redis_role }}_replica.conf"
   notify: restart redis
   register: updated_config
   when: "'db-redis-story2' not in inventory_hostname"
@@ -46,7 +61,7 @@
     
 - name: Create Redis docker volume with correct permissions
   file:
-    path: /srv/newsblur/docker/volumes/redis
+    path: "/srv/newsblur/docker/volumes/redis_{{ redis_role }}"
     state: directory
     recurse: yes
     owner: "{{ ansible_effective_user_id|int }}"
@@ -54,7 +69,7 @@
 
 - name: Start redis docker containers
   docker_container:
-    name: redis
+    name: "redis-{{ redis_role }}"
     image: redis:7
     pull: true
     state: started
@@ -69,19 +84,19 @@
         aliases: 
           - redis
     ports:
-      - 6379:6379
+      - "{{ redis_port }}:6379"
     restart_policy: unless-stopped
     user: "{{ ansible_effective_user_id|int }}:{{ ansible_effective_group_id|int }}"
     volumes:
-      - /srv/newsblur/docker/volumes/redis:/data
+      - "/srv/newsblur/docker/volumes/redis_{{ redis_role }}:/data"
       - /srv/newsblur/docker/redis/redis.conf:/usr/local/etc/redis/redis_server.conf
-      - /srv/newsblur/docker/redis/redis_replica.conf:/usr/local/etc/redis/redis_replica.conf
+      - "/srv/newsblur/docker/redis/redis_{{ redis_role }}_replica.conf:/usr/local/etc/redis/redis_replica.conf"
 
 - name: Register redis in consul
   become: yes
   template:
     src: consul_service.json
-    dest: /etc/consul.d/redis.json
+    dest: "/etc/consul.d/redis_{{ redis_role }}.json"
   notify:
     - reload consul
   tags: consul
@@ -130,7 +145,7 @@
     job: >
       docker run --rm
       -v /srv/newsblur:/srv/newsblur
-      -v /srv/newsblur/docker/volumes/redis/dump.rdb:/data/dump.rdb
+      -v /srv/newsblur/docker/volumes/redis_{{ redis_role }}/dump.rdb:/data/dump.rdb
       --network=newsblurnet
       --hostname={{ ansible_hostname }}
       newsblur/newsblur_python3 python /srv/newsblur/utils/backups/backup_redis.py

ansible/roles/redis/templates/consul_service.json

@@ -1,25 +1,25 @@
 {
     "service": {
         {% if inventory_hostname in ["db-redis-user", "db-redis-story1", "db-redis-session", "db-redis-pubsub"] %}
-            "name": "{{ inventory_hostname|regex_replace('\d+', '') }}",
+            "name": "{{ inventory_hostname|regex_replace('\-?\d+', '')|regex_replace("hdb", "db") }}",
         {% else %}
-            "name": "{{ inventory_hostname|regex_replace('\-?\d+', '')|regex_replace('hdb-', 'db-') }}-staging",
+            "name": "db-redis-{{ redis_role }}-staging",
         {% endif %}
         "id": "{{ inventory_hostname }}",
         "tags": [
             "redis"
         ],
-        "port": 6379,
+        "port": {{ redis_port }},
         "checks": [{
             "id": "{{inventory_hostname}}-ping",
             {% if 'db-redis-story' in inventory_hostname %}
-                "http": "http://{{ ansible_host }}:5579/db_check/redis_story?consul=1",
+                "http": "http://{{ ansible_host }}:5579/db_check/redis_story?consul=1&port={{ redis_port }}",
             {% elif 'db-redis-user' in inventory_hostname %}
-                "http": "http://{{ ansible_host }}:5579/db_check/redis_user?consul=1",
+                "http": "http://{{ ansible_host }}:5579/db_check/redis_user?consul=1&port={{ redis_port }}",
             {% elif 'db-redis-pubsub' in inventory_hostname %}
-                "http": "http://{{ ansible_host }}:5579/db_check/redis_pubsub?consul=1",
-            {% elif 'db-redis-sessions' in inventory_hostname %}
-                "http": "http://{{ ansible_host }}:5579/db_check/redis_sessions?consul=1",            
+                "http": "http://{{ ansible_host }}:5579/db_check/redis_pubsub?consul=1&port={{ redis_port }}",
+            {% elif 'db-redis-session' in inventory_hostname %}
+                "http": "http://{{ ansible_host }}:5579/db_check/redis_sessions?consul=1&port={{ redis_port }}",
             {% else %}
                 "http": "http://{{ ansible_host }}:5000/db_check/redis?consul=1",
             {% endif %}

flask_monitor/db_monitor.py

@@ -156,8 +156,10 @@ def db_check_redis_user():
     if request.args.get('consul') == '1':
         return str(1)
 
+    port = request.args.get('port', 5379)
+
     try:
-        r = redis.Redis(f'{settings.SERVER_NAME}.node.nyc1.consul', db=0)
+        r = redis.Redis(f'{settings.SERVER_NAME}.node.nyc1.consul:{port}', db=0)
     except:
         abort(Response("Can't connect to db", 503))
     
@@ -176,8 +178,10 @@ def db_check_redis_story():
     if request.args.get('consul') == '1':
         return str(1)
 
+    port = request.args.get('port', 5379)
+    
     try:
-        r = redis.Redis(f'{settings.SERVER_NAME}.node.nyc1.consul', db=1)
+        r = redis.Redis(f'{settings.SERVER_NAME}.node.nyc1.consul:{port}', db=1)
     except:
         abort(Response("Can't connect to db", 503))
     
@@ -196,8 +200,10 @@ def db_check_redis_sessions():
     if request.args.get('consul') == '1':
         return str(1)
 
+    port = request.args.get('port', 5379)
+
     try:
-        r = redis.Redis(f'{settings.SERVER_NAME}.node.nyc1.consul', db=5)
+        r = redis.Redis(f'{settings.SERVER_NAME}.node.nyc1.consul:{port}', db=5)
     except:
         abort(Response("Can't connect to db", 503))
     
@@ -216,8 +222,10 @@ def db_check_redis_pubsub():
     if request.args.get('consul') == '1':
         return str(1)
 
+    port = request.args.get('port', 5379)
+
     try:
-        r = redis.Redis(f'{settings.SERVER_NAME}.node.nyc1.consul', db=1)
+        r = redis.Redis(f'{settings.SERVER_NAME}.node.nyc1.consul:{port}', db=1)
     except:
         abort(Response("Can't connect to db", 503))