Updating postgres for standby.

Maintenance.md

@@ -176,3 +176,17 @@ Provision a new redis server, replicate the data, take newsblur down for mainten
    aps -l db-redis-story1,db-redis-story2 -t consul
    make maintenance_off
    make task
+
+### Switching to a new postgres server
+
+   # Old
+   docker exec -it -u postgres postgres psql -c "SELECT pg_start_backup('label', true)"
+   # New
+   ## Install `openssh-client` and `rsync`
+   docker stop postgres
+   rsync -Pav --stats --progress db-postgres.service.consul:/srv/newsblur/docker/volumes/postgres/data /srv/newsblur/docker/volumes/postgres/ --exclude postmaster.pid
+   docker start postgres
+   # New
+   docker exec -it -u postgres postgres /usr/lib/postgresql/13/bin/pg_ctl -D /var/lib/postgresql/data promote
+   # Old
+   docker exec -it -u postgres postgres psql -c "SELECT pg_stop_backup()"

ansible/roles/postgres/handlers/main.yml

@@ -10,5 +10,5 @@
 
 - name: reload postgres config
   become: yes
-  command: docker exec -u postgres postgres pg_ctl reload
+  command: docker exec postgres pg_ctl reload
   listen: reload postgres

ansible/roles/postgres/tasks/main.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Template postgresql-13.conf file
   template:
     src: /srv/newsblur/docker/postgres/postgresql-13.conf.j2
@@ -8,22 +7,60 @@
   register: updated_config
 
 - name: Create Postgres docker volumes with correct permissions
-  become: yes
   file:
     path: "{{ item  }}"
     state: directory
     recurse: yes
     owner: "{{ ansible_effective_user_id|int }}"
-    group: "{{ ansible_effective_group_id|int }}"  
+    group: "{{ ansible_effective_group_id|int }}"
   with_items:
     - /srv/newsblur/docker/volumes/postgres/archive
     - /srv/newsblur/docker/volumes/postgres/backups
     - /srv/newsblur/docker/volumes/postgres/data
 
+- name: Template postgres secondaries with empty standby.signal file
+  file:
+    path: /srv/newsblur/docker/volumes/postgres/data/standby.signal
+    state: file
+    owner: "{{ ansible_effective_user_id|int }}"
+    group: "{{ ansible_effective_group_id|int }}"
+  when: (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ['db-postgres-secondary', 'hdb-postgres']
+
+- name: Copy SSH private key
+  copy:
+    src: /srv/secrets-newsblur/keys/postgres.key
+    dest: /home/nb/.ssh/id_rsa
+    owner: "{{ ansible_effective_user_id|int }}"
+    group: "{{ ansible_effective_group_id|int }}"
+    mode: "0600"
+
+- name: Copy SSH public key
+  copy:
+    src: /srv/secrets-newsblur/keys/postgres.key.pub
+    dest: /home/nb/.ssh/id_rsa.pub
+    owner: "{{ ansible_effective_user_id|int }}"
+    group: "{{ ansible_effective_group_id|int }}"
+    mode: "0600"
+
+- name: Add SSH public key to authorized keys
+  authorized_key:
+    user: "nb"
+    state: present
+    key: "{{ lookup('file', '/srv/secrets-newsblur/keys/postgres.key.pub') }}"
+
+- name: Build the custom postgres docker image
+  docker_image:
+    name: newsblur/postgres:13
+    build:
+      path: /srv/newsblur/docker/postgres/Dockerfile.postgres
+      pull: yes
+    force_tag: yes
+    state: present
+
 - name: Start postgres docker containers
   docker_container:
     name: postgres
-    image: postgres:13
+    image: newsblur/postgres:13
     state: started
     container_default_behavior: no_defaults
     command: postgres -c config_file=/etc/postgresql/postgresql.conf
@@ -35,11 +72,11 @@
     network_mode: default
     networks:
       - name: newsblurnet
-        aliases: 
+        aliases:
           - postgres
     ports:
       - 5432:5432
-    user: 1000:1001
+    user: "{{ ansible_effective_user_id|int }}:{{ ansible_effective_group_id|int }}"
     volumes:
       - /srv/newsblur/docker/volumes/postgres/data:/var/lib/postgresql/data
       - /srv/newsblur/docker/volumes/postgres/archive:/var/lib/postgresql/archive
@@ -47,15 +84,16 @@
       - /srv/newsblur/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf
       - /srv/newsblur/docker/postgres/postgres_hba-13.conf:/etc/postgresql/pg_hba.conf
       - /srv/newsblur/docker/postgres/postgres_ident-13.conf:/etc/postgresql/pg_ident.conf
+      - /home/nb/.ssh/id_rsa:/var/lib/postgresql/.ssh/id_rsa
     restart_policy: unless-stopped
   when: (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ['db-postgres-primary', 'db-postgres', 'hdb-postgres']
 
-- name: Change ownership in postgres docker container
-  become: yes
-  command: >
-    docker exec postgres chown -fR postgres.postgres /var/lib/postgresql
-  ignore_errors: yes
-  
+# - name: Change ownership in postgres docker container
+#   become: yes
+#   command: >
+#     docker exec postgres chown -fR postgres.postgres /var/lib/postgresql
+#   ignore_errors: yes
+
 - name: Ensure newsblur role in postgres
   become: yes
   shell: >
@@ -63,9 +101,10 @@
     docker exec postgres createuser -s newsblur -U postgres;
     docker exec postgres createdb newsblur -U newsblur;
   register: ensure_role
+  ignore_errors: yes
   changed_when:
-      - "ensure_role.rc == 0"
-  failed_when: 
+    - "ensure_role.rc == 0"
+  failed_when:
     - "'already exists' not in ensure_role.stderr"
     - "ensure_role.rc != 0"
 
@@ -89,7 +128,9 @@
     name: disk_usage_sanity_checker
     minute: "0"
     job: >-
-      OUTPUT=$(df / | head -n 2 |  tail -1) docker run --rm -it -v /srv/newsblur:/srv/newsblur --network=newsblurnet --hostname {{ ansible_hostname }} newsblur/newsblur_python3 /srv/newsblur/utils/monitor_disk_usage.py $OUTPUT
+      OUTPUT=$(df / | head -n 2 |  tail -1) docker run --rm -it -v /srv/newsblur:/srv/newsblur \
+      --network=newsblurnet --hostname {{ ansible_hostname }} newsblur/newsblur_python3 \
+      /srv/newsblur/utils/monitor_disk_usage.py $OUTPUT
   tags: cron
 
 - name: Add postgresql archive cleaner cronjob
@@ -115,8 +156,8 @@
     path: /var/log/postgres_backup.log
     state: touch
     mode: 0777
-    owner: 1000
-    group: 1001
+    owner: "{{ ansible_effective_user_id|int }}"
+    group: "{{ ansible_effective_group_id|int }}"
 
 - name: Add postgres backup
   cron:
@@ -125,4 +166,3 @@
     hour: "4"
     job: /srv/newsblur/docker/postgres/backup_postgres.sh >> /var/log/postgres_backup.log 2>&1
   tags: cron
-

config/postgresql_recovery.conf

@@ -6,7 +6,7 @@ standby_mode          = 'on'
 
 # Specifies a connection string which is used for the standby server to connect
 # with the primary.
-primary_conninfo      = 'host=db_pgsql port=5432 user=postgres'
+primary_conninfo      = 'host=db-postgres.service.consul port=5432 user=postgres'
 
 # Specifies a trigger file whose presence should cause streaming replication to
 # end (i.e., failover).
@@ -17,4 +17,4 @@ trigger_file = '/var/lib/postgresql/11/main/standby.trigger'
 # required for the standby server, this may not be necessary. But
 # a large workload can cause segments to be recycled before the standby
 # is fully synchronized, requiring you to start again from a new base backup.
-restore_command = 'rsync -a db_pgsql:/var/lib/postgresql/13/archive/%f "%p"'
+restore_command = 'rsync -a db-postgres.service.consul:/var/lib/postgresql/13/archive/%f "%p"'

docker/postgres/Dockerfile.postgres

@@ -0,0 +1,4 @@
+FROM postgres:13
+
+RUN apt update && apt install -y openssh-client rsync
+

docker/postgres/postgresql-13.conf.j2

@@ -1,3 +1,6 @@
+{% if (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ["hdb-postgres", "db-postgres-secondary"] %}
+primary_conninfo      = 'host=db-postgres.service.consul port=5432 user=postgres'
+{% endif %}
 # -----------------------------
 # PostgreSQL configuration file
 # -----------------------------
@@ -246,6 +249,7 @@ archive_command = 'test ! -f /var/lib/postgresql/archive/%f && cp -f %p /var/lib
 
 # These are only used in recovery mode.
 
+# restore_command = 'rsync -a -e "ssh -i /var/lib/postgresql/.ssh/id_rsa" db-postgres.service.consul:/srv/newsblur/docker/volumes/postgres/archive/%f "%p"'
 restore_command = 'cp /var/lib/postgresql/archive/%f %p'		# command to use to restore an archived logfile segment
 				# placeholders: %p = path of file to restore
 				#               %f = file name only