From 1453c3ee273ac1b0d69a40f35cd9f05b4eae718f Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sat, 27 Jan 2024 23:11:36 -0500 Subject: [PATCH] Updating postgres for standby. --- Maintenance.md | 14 +++++ ansible/roles/postgres/handlers/main.yml | 2 +- ansible/roles/postgres/tasks/main.yml | 76 ++++++++++++++++++------ config/postgresql_recovery.conf | 4 +- docker/postgres/Dockerfile.postgres | 4 ++ docker/postgres/postgresql-13.conf.j2 | 4 ++ 6 files changed, 83 insertions(+), 21 deletions(-) create mode 100644 docker/postgres/Dockerfile.postgres diff --git a/Maintenance.md b/Maintenance.md index 3230707cb..bb3a1d0a5 100644 --- a/Maintenance.md +++ b/Maintenance.md @@ -176,3 +176,17 @@ Provision a new redis server, replicate the data, take newsblur down for mainten aps -l db-redis-story1,db-redis-story2 -t consul make maintenance_off make task + +### Switching to a new postgres server + + # Old + docker exec -it -u postgres postgres psql -c "SELECT pg_start_backup('label', true)" + # New + ## Install `openssh-client` and `rsync` + docker stop postgres + rsync -Pav --stats --progress db-postgres.service.consul:/srv/newsblur/docker/volumes/postgres/data /srv/newsblur/docker/volumes/postgres/ --exclude postmaster.pid + docker start postgres + # New + docker exec -it -u postgres postgres /usr/lib/postgresql/13/bin/pg_ctl -D /var/lib/postgresql/data promote + # Old + docker exec -it -u postgres postgres psql -c "SELECT pg_stop_backup()" diff --git a/ansible/roles/postgres/handlers/main.yml b/ansible/roles/postgres/handlers/main.yml index bd8210f9c..c201266d9 100644 --- a/ansible/roles/postgres/handlers/main.yml +++ b/ansible/roles/postgres/handlers/main.yml @@ -10,5 +10,5 @@ - name: reload postgres config become: yes - command: docker exec -u postgres postgres pg_ctl reload + command: docker exec postgres pg_ctl reload listen: reload postgres diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml index 03bf93903..3fff606ad 100644 --- a/ansible/roles/postgres/tasks/main.yml +++ b/ansible/roles/postgres/tasks/main.yml @@ -1,5 +1,4 @@ --- - - name: Template postgresql-13.conf file template: src: /srv/newsblur/docker/postgres/postgresql-13.conf.j2 @@ -8,22 +7,60 @@ register: updated_config - name: Create Postgres docker volumes with correct permissions - become: yes file: path: "{{ item }}" state: directory recurse: yes owner: "{{ ansible_effective_user_id|int }}" - group: "{{ ansible_effective_group_id|int }}" + group: "{{ ansible_effective_group_id|int }}" with_items: - /srv/newsblur/docker/volumes/postgres/archive - /srv/newsblur/docker/volumes/postgres/backups - /srv/newsblur/docker/volumes/postgres/data +- name: Template postgres secondaries with empty standby.signal file + file: + path: /srv/newsblur/docker/volumes/postgres/data/standby.signal + state: file + owner: "{{ ansible_effective_user_id|int }}" + group: "{{ ansible_effective_group_id|int }}" + when: (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ['db-postgres-secondary', 'hdb-postgres'] + +- name: Copy SSH private key + copy: + src: /srv/secrets-newsblur/keys/postgres.key + dest: /home/nb/.ssh/id_rsa + owner: "{{ ansible_effective_user_id|int }}" + group: "{{ ansible_effective_group_id|int }}" + mode: "0600" + +- name: Copy SSH public key + copy: + src: /srv/secrets-newsblur/keys/postgres.key.pub + dest: /home/nb/.ssh/id_rsa.pub + owner: "{{ ansible_effective_user_id|int }}" + group: "{{ ansible_effective_group_id|int }}" + mode: "0600" + +- name: Add SSH public key to authorized keys + authorized_key: + user: "nb" + state: present + key: "{{ lookup('file', '/srv/secrets-newsblur/keys/postgres.key.pub') }}" + +- name: Build the custom postgres docker image + docker_image: + name: newsblur/postgres:13 + build: + path: /srv/newsblur/docker/postgres/Dockerfile.postgres + pull: yes + force_tag: yes + state: present + - name: Start postgres docker containers docker_container: name: postgres - image: postgres:13 + image: newsblur/postgres:13 state: started container_default_behavior: no_defaults command: postgres -c config_file=/etc/postgresql/postgresql.conf @@ -35,11 +72,11 @@ network_mode: default networks: - name: newsblurnet - aliases: + aliases: - postgres ports: - 5432:5432 - user: 1000:1001 + user: "{{ ansible_effective_user_id|int }}:{{ ansible_effective_group_id|int }}" volumes: - /srv/newsblur/docker/volumes/postgres/data:/var/lib/postgresql/data - /srv/newsblur/docker/volumes/postgres/archive:/var/lib/postgresql/archive @@ -47,15 +84,16 @@ - /srv/newsblur/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf - /srv/newsblur/docker/postgres/postgres_hba-13.conf:/etc/postgresql/pg_hba.conf - /srv/newsblur/docker/postgres/postgres_ident-13.conf:/etc/postgresql/pg_ident.conf + - /home/nb/.ssh/id_rsa:/var/lib/postgresql/.ssh/id_rsa restart_policy: unless-stopped when: (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ['db-postgres-primary', 'db-postgres', 'hdb-postgres'] -- name: Change ownership in postgres docker container - become: yes - command: > - docker exec postgres chown -fR postgres.postgres /var/lib/postgresql - ignore_errors: yes - +# - name: Change ownership in postgres docker container +# become: yes +# command: > +# docker exec postgres chown -fR postgres.postgres /var/lib/postgresql +# ignore_errors: yes + - name: Ensure newsblur role in postgres become: yes shell: > @@ -63,9 +101,10 @@ docker exec postgres createuser -s newsblur -U postgres; docker exec postgres createdb newsblur -U newsblur; register: ensure_role + ignore_errors: yes changed_when: - - "ensure_role.rc == 0" - failed_when: + - "ensure_role.rc == 0" + failed_when: - "'already exists' not in ensure_role.stderr" - "ensure_role.rc != 0" @@ -89,7 +128,9 @@ name: disk_usage_sanity_checker minute: "0" job: >- - OUTPUT=$(df / | head -n 2 | tail -1) docker run --rm -it -v /srv/newsblur:/srv/newsblur --network=newsblurnet --hostname {{ ansible_hostname }} newsblur/newsblur_python3 /srv/newsblur/utils/monitor_disk_usage.py $OUTPUT + OUTPUT=$(df / | head -n 2 | tail -1) docker run --rm -it -v /srv/newsblur:/srv/newsblur \ + --network=newsblurnet --hostname {{ ansible_hostname }} newsblur/newsblur_python3 \ + /srv/newsblur/utils/monitor_disk_usage.py $OUTPUT tags: cron - name: Add postgresql archive cleaner cronjob @@ -115,8 +156,8 @@ path: /var/log/postgres_backup.log state: touch mode: 0777 - owner: 1000 - group: 1001 + owner: "{{ ansible_effective_user_id|int }}" + group: "{{ ansible_effective_group_id|int }}" - name: Add postgres backup cron: @@ -125,4 +166,3 @@ hour: "4" job: /srv/newsblur/docker/postgres/backup_postgres.sh >> /var/log/postgres_backup.log 2>&1 tags: cron - diff --git a/config/postgresql_recovery.conf b/config/postgresql_recovery.conf index 9e35d9828..8da8eb674 100644 --- a/config/postgresql_recovery.conf +++ b/config/postgresql_recovery.conf @@ -6,7 +6,7 @@ standby_mode = 'on' # Specifies a connection string which is used for the standby server to connect # with the primary. -primary_conninfo = 'host=db_pgsql port=5432 user=postgres' +primary_conninfo = 'host=db-postgres.service.consul port=5432 user=postgres' # Specifies a trigger file whose presence should cause streaming replication to # end (i.e., failover). @@ -17,4 +17,4 @@ trigger_file = '/var/lib/postgresql/11/main/standby.trigger' # required for the standby server, this may not be necessary. But # a large workload can cause segments to be recycled before the standby # is fully synchronized, requiring you to start again from a new base backup. -restore_command = 'rsync -a db_pgsql:/var/lib/postgresql/13/archive/%f "%p"' +restore_command = 'rsync -a db-postgres.service.consul:/var/lib/postgresql/13/archive/%f "%p"' diff --git a/docker/postgres/Dockerfile.postgres b/docker/postgres/Dockerfile.postgres new file mode 100644 index 000000000..4a18a602b --- /dev/null +++ b/docker/postgres/Dockerfile.postgres @@ -0,0 +1,4 @@ +FROM postgres:13 + +RUN apt update && apt install -y openssh-client rsync + diff --git a/docker/postgres/postgresql-13.conf.j2 b/docker/postgres/postgresql-13.conf.j2 index 46ef56aa6..108c23399 100644 --- a/docker/postgres/postgresql-13.conf.j2 +++ b/docker/postgres/postgresql-13.conf.j2 @@ -1,3 +1,6 @@ +{% if (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ["hdb-postgres", "db-postgres-secondary"] %} +primary_conninfo = 'host=db-postgres.service.consul port=5432 user=postgres' +{% endif %} # ----------------------------- # PostgreSQL configuration file # ----------------------------- @@ -246,6 +249,7 @@ archive_command = 'test ! -f /var/lib/postgresql/archive/%f && cp -f %p /var/lib # These are only used in recovery mode. +# restore_command = 'rsync -a -e "ssh -i /var/lib/postgresql/.ssh/id_rsa" db-postgres.service.consul:/srv/newsblur/docker/volumes/postgres/archive/%f "%p"' restore_command = 'cp /var/lib/postgresql/archive/%f %p' # command to use to restore an archived logfile segment # placeholders: %p = path of file to restore # %f = file name only