From 464fecd18e8a412fe34615378c88324de301b151 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 29 Apr 2022 16:44:46 -0400 Subject: [PATCH 1/7] Adding postgres restore command and updating backup script. --- ansible/roles/sentry/handlers/main.yml | 6 ------ ansible/roles/sentry/tasks/main.yml | 11 ++++++++++- docker/postgres/backup_postgres.sh | 10 +++++----- docker/postgres/postgresql-13.conf.j2 | 2 +- utils/backups/backup_psql.py | 6 +++--- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/ansible/roles/sentry/handlers/main.yml b/ansible/roles/sentry/handlers/main.yml index c59af61eb..ed97d539c 100644 --- a/ansible/roles/sentry/handlers/main.yml +++ b/ansible/roles/sentry/handlers/main.yml @@ -1,7 +1 @@ --- -- name: reload sentry - become: yes - command: - chdir: /srv/sentry/ - cmd: ./install.sh - listen: reload sentry diff --git a/ansible/roles/sentry/tasks/main.yml b/ansible/roles/sentry/tasks/main.yml index d735295c4..0f1eeb5df 100644 --- a/ansible/roles/sentry/tasks/main.yml +++ b/ansible/roles/sentry/tasks/main.yml @@ -4,7 +4,16 @@ repo: https://github.com/getsentry/self-hosted.git dest: /srv/sentry/ version: master - notify: reload sentry + +- name: Updating Sentry + command: + chdir: /srv/sentry/ + cmd: ./install.sh + +- name: docker-compuse up -d + command: + chdir: /srv/sentry/ + cmd: docker-compose up -d - name: Register sentry in consul tags: consul diff --git a/docker/postgres/backup_postgres.sh b/docker/postgres/backup_postgres.sh index 97d5c8b16..a56935511 100755 --- a/docker/postgres/backup_postgres.sh +++ b/docker/postgres/backup_postgres.sh @@ -2,18 +2,18 @@ now=$(date '+%Y-%m-%d-%H-%M') -echo "---> PG dumping - ${now}" -BACKUP_FILE="/srv/newsblur/backup/backup_postgresql_${now}.sql" -sudo docker exec -it postgres /usr/lib/postgresql/13/bin/pg_dump -U newsblur -h 127.0.0.1 -Fc newsblur > $BACKUP_FILE +BACKUP_FILE="/var/lib/postgresql/backup/backup_postgresql_${now}.sql" +echo "---> PG dumping - ${now}: ${BACKUP_FILE}" +sudo docker exec -it postgres sh -c '/usr/lib/postgresql/13/bin/pg_dump -U newsblur -h 127.0.0.1 -Fc newsblur > ${BACKUP_FILE}' echo " ---> Uploading postgres backup to S3" sudo docker run --user 1000:1001 --rm \ -v /srv/newsblur:/srv/newsblur \ --network=host \ newsblur/newsblur_python3 \ - python /srv/newsblur/utils/backups/backup_psql.py + python /srv/newsblur/utils/backups/backup_psql.py $BACKUP_FILE # Don't delete backup since the backup_mongo.py script will rm them ## rm /opt/mongo/newsblur/backup/backup_mongo_${now}.tgz ## rm /opt/mongo/newsblur/backup/backup_mongo_${now} -echo " ---> Finished uploading backups to S3: " +echo " ---> Finished uploading backups to S3" diff --git a/docker/postgres/postgresql-13.conf.j2 b/docker/postgres/postgresql-13.conf.j2 index 5ee4d01f2..ae09de08f 100644 --- a/docker/postgres/postgresql-13.conf.j2 +++ b/docker/postgres/postgresql-13.conf.j2 @@ -246,7 +246,7 @@ archive_command = 'test ! -f /var/lib/postgresql/archive/%f && cp -f %p /var/lib # These are only used in recovery mode. -#restore_command = '' # command to use to restore an archived logfile segment +restore_command = 'cp /var/lib/postgresql/archive/%f %p' # command to use to restore an archived logfile segment # placeholders: %p = path of file to restore # %f = file name only # e.g. 'cp /mnt/server/archivedir/%f %p' diff --git a/utils/backups/backup_psql.py b/utils/backups/backup_psql.py index 86ecab271..78ddcf226 100644 --- a/utils/backups/backup_psql.py +++ b/utils/backups/backup_psql.py @@ -37,9 +37,9 @@ BACKUP_DIR = '/srv/newsblur/backup/' s3 = boto3.client('s3', aws_access_key_id=settings.S3_ACCESS_KEY, aws_secret_access_key=settings.S3_SECRET) hostname = socket.gethostname().replace('-','_') -s3_object_name = f'backup_{hostname}/backup_{hostname}_{time.strftime("%Y-%m-%d-%H-%M")}.sql' -path = os.listdir(BACKUP_DIR)[0] -full_path = os.path.join(BACKUP_DIR, path) +full_path = sys.argv[1] +backup_filename = os.path.basename(full_path) +s3_object_name = f'backup_{hostname}/backup_{hostname}_{backup_filename}.sql' print('Uploading %s to %s on S3 bucket %s' % (full_path, s3_object_name, settings.S3_BACKUP_BUCKET)) s3.upload_file(full_path, settings.S3_BACKUP_BUCKET, s3_object_name, Callback=ProgressPercentage(full_path)) os.remove(full_path) From 1dc0e98448ac0005141dc122112054042c7d9f47 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 29 Apr 2022 16:53:47 -0400 Subject: [PATCH 2/7] Renaming postgresql backups --- docker/postgres/backup_postgres.sh | 8 +++++--- utils/backups/backup_psql.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/postgres/backup_postgres.sh b/docker/postgres/backup_postgres.sh index a56935511..c5dd349d7 100755 --- a/docker/postgres/backup_postgres.sh +++ b/docker/postgres/backup_postgres.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash now=$(date '+%Y-%m-%d-%H-%M') - -BACKUP_FILE="/var/lib/postgresql/backup/backup_postgresql_${now}.sql" +BACKUP_PATH="/var/lib/postgresql/backup/" +BACKUP_FILE="${BACKUP_PATH}backup_postgresql_${now}.sql" echo "---> PG dumping - ${now}: ${BACKUP_FILE}" -sudo docker exec -it postgres sh -c '/usr/lib/postgresql/13/bin/pg_dump -U newsblur -h 127.0.0.1 -Fc newsblur > ${BACKUP_FILE}' +sudo docker exec -it postgres sh -c "mkdir -p $BACKUP_PATH" +sudo docker exec -it postgres sh -c "/usr/lib/postgresql/13/bin/pg_dump -U newsblur -h 127.0.0.1 -Fc newsblur > $BACKUP_FILE" + echo " ---> Uploading postgres backup to S3" sudo docker run --user 1000:1001 --rm \ diff --git a/utils/backups/backup_psql.py b/utils/backups/backup_psql.py index 78ddcf226..6fec0466e 100644 --- a/utils/backups/backup_psql.py +++ b/utils/backups/backup_psql.py @@ -39,7 +39,7 @@ s3 = boto3.client('s3', aws_access_key_id=settings.S3_ACCESS_KEY, aws_secret_acc hostname = socket.gethostname().replace('-','_') full_path = sys.argv[1] backup_filename = os.path.basename(full_path) -s3_object_name = f'backup_{hostname}/backup_{hostname}_{backup_filename}.sql' +s3_object_name = f'backup_{hostname}/{backup_filename}.sql' print('Uploading %s to %s on S3 bucket %s' % (full_path, s3_object_name, settings.S3_BACKUP_BUCKET)) s3.upload_file(full_path, settings.S3_BACKUP_BUCKET, s3_object_name, Callback=ProgressPercentage(full_path)) os.remove(full_path) From 94fda89ee5b830e25ad237e1f18736b04621f96b Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 29 Apr 2022 17:13:17 -0400 Subject: [PATCH 3/7] Paths are in different environments. --- docker/postgres/backup_postgres.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/postgres/backup_postgres.sh b/docker/postgres/backup_postgres.sh index c5dd349d7..b4e3e4ca6 100755 --- a/docker/postgres/backup_postgres.sh +++ b/docker/postgres/backup_postgres.sh @@ -2,7 +2,10 @@ now=$(date '+%Y-%m-%d-%H-%M') BACKUP_PATH="/var/lib/postgresql/backup/" -BACKUP_FILE="${BACKUP_PATH}backup_postgresql_${now}.sql" +UPLOAD_PATH="/srv/newsblur/backups/" +BACKUP_FILENAME="backup_postgresql_${now}.sql" +BACKUP_FILE="${BACKUP_PATH}${BACKUP_FILENAME}" +UPLOAD_FILE="${UPLOAD_PATH}${BACKUP_FILENAME}" echo "---> PG dumping - ${now}: ${BACKUP_FILE}" sudo docker exec -it postgres sh -c "mkdir -p $BACKUP_PATH" sudo docker exec -it postgres sh -c "/usr/lib/postgresql/13/bin/pg_dump -U newsblur -h 127.0.0.1 -Fc newsblur > $BACKUP_FILE" @@ -13,7 +16,7 @@ sudo docker run --user 1000:1001 --rm \ -v /srv/newsblur:/srv/newsblur \ --network=host \ newsblur/newsblur_python3 \ - python /srv/newsblur/utils/backups/backup_psql.py $BACKUP_FILE + python /srv/newsblur/utils/backups/backup_psql.py $UPLOAD_FILE # Don't delete backup since the backup_mongo.py script will rm them ## rm /opt/mongo/newsblur/backup/backup_mongo_${now}.tgz From 9ea16aadb7581edc7d16bd68dc916f72a20ede3a Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sat, 30 Apr 2022 06:47:09 -0400 Subject: [PATCH 4/7] No TTY/input needed on backups. --- ansible/roles/postgres/handlers/main.yml | 2 +- docker/postgres/backup_postgres.sh | 4 ++-- docker/postgres/postgresql-13.conf.j2 | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/roles/postgres/handlers/main.yml b/ansible/roles/postgres/handlers/main.yml index c201266d9..bd8210f9c 100644 --- a/ansible/roles/postgres/handlers/main.yml +++ b/ansible/roles/postgres/handlers/main.yml @@ -10,5 +10,5 @@ - name: reload postgres config become: yes - command: docker exec postgres pg_ctl reload + command: docker exec -u postgres postgres pg_ctl reload listen: reload postgres diff --git a/docker/postgres/backup_postgres.sh b/docker/postgres/backup_postgres.sh index b4e3e4ca6..0293a0b19 100755 --- a/docker/postgres/backup_postgres.sh +++ b/docker/postgres/backup_postgres.sh @@ -7,8 +7,8 @@ BACKUP_FILENAME="backup_postgresql_${now}.sql" BACKUP_FILE="${BACKUP_PATH}${BACKUP_FILENAME}" UPLOAD_FILE="${UPLOAD_PATH}${BACKUP_FILENAME}" echo "---> PG dumping - ${now}: ${BACKUP_FILE}" -sudo docker exec -it postgres sh -c "mkdir -p $BACKUP_PATH" -sudo docker exec -it postgres sh -c "/usr/lib/postgresql/13/bin/pg_dump -U newsblur -h 127.0.0.1 -Fc newsblur > $BACKUP_FILE" +sudo docker exec postgres sh -c "mkdir -p $BACKUP_PATH" +sudo docker exec postgres sh -c "/usr/lib/postgresql/13/bin/pg_dump -U newsblur -h 127.0.0.1 -Fc newsblur > $BACKUP_FILE" echo " ---> Uploading postgres backup to S3" diff --git a/docker/postgres/postgresql-13.conf.j2 b/docker/postgres/postgresql-13.conf.j2 index ae09de08f..60309c088 100644 --- a/docker/postgres/postgresql-13.conf.j2 +++ b/docker/postgres/postgresql-13.conf.j2 @@ -38,7 +38,7 @@ # The default values of these variables are driven from the -D command-line # option or PGDATA environment variable, represented here as ConfigDir. -data_directory = '/var/lib/postgresql/data' # use data in another directory +data_directory = '/var/lib/postgresql/main' # use data in another directory # (change requires restart) hba_file = '/etc/postgresql/pg_hba.conf' # host-based authentication file # (change requires restart) From de309183c5a2f8df3d977de49df2b5b670a74d21 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sat, 30 Apr 2022 06:51:11 -0400 Subject: [PATCH 5/7] Adding timestamps to backup logs --- ansible/roles/postgres/tasks/main.yml | 2 +- docker/postgres/backup_postgres.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml index 54f282fe0..1c7f6773e 100644 --- a/ansible/roles/postgres/tasks/main.yml +++ b/ansible/roles/postgres/tasks/main.yml @@ -105,5 +105,5 @@ name: postgres backup minute: "0" hour: "4" - job: /srv/newsblur/docker/postgres/backup_postgres.sh 1> /var/log/postgres_backup.log 2>&1 + job: /srv/newsblur/docker/postgres/backup_postgres.sh >> /var/log/postgres_backup.log 2>&1 diff --git a/docker/postgres/backup_postgres.sh b/docker/postgres/backup_postgres.sh index 0293a0b19..df58d450d 100755 --- a/docker/postgres/backup_postgres.sh +++ b/docker/postgres/backup_postgres.sh @@ -6,12 +6,12 @@ UPLOAD_PATH="/srv/newsblur/backups/" BACKUP_FILENAME="backup_postgresql_${now}.sql" BACKUP_FILE="${BACKUP_PATH}${BACKUP_FILENAME}" UPLOAD_FILE="${UPLOAD_PATH}${BACKUP_FILENAME}" -echo "---> PG dumping - ${now}: ${BACKUP_FILE}" +echo $(date -u) "---> PG dumping - ${now}: ${BACKUP_FILE}" sudo docker exec postgres sh -c "mkdir -p $BACKUP_PATH" sudo docker exec postgres sh -c "/usr/lib/postgresql/13/bin/pg_dump -U newsblur -h 127.0.0.1 -Fc newsblur > $BACKUP_FILE" -echo " ---> Uploading postgres backup to S3" +echo $(date -u) " ---> Uploading postgres backup to S3" sudo docker run --user 1000:1001 --rm \ -v /srv/newsblur:/srv/newsblur \ --network=host \ @@ -21,4 +21,4 @@ sudo docker run --user 1000:1001 --rm \ # Don't delete backup since the backup_mongo.py script will rm them ## rm /opt/mongo/newsblur/backup/backup_mongo_${now}.tgz ## rm /opt/mongo/newsblur/backup/backup_mongo_${now} -echo " ---> Finished uploading backups to S3" +echo $(date -u) " ---> Finished uploading backups to S3" From 3670faf99d0b407f92ff600efda7e317de92e2da Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sat, 30 Apr 2022 07:24:46 -0400 Subject: [PATCH 6/7] Adding a secondary postgres that starts from a pg_basebackup. Still needs testing. --- ansible/roles/postgres/tasks/main.yml | 40 +++++++++++++++++++++------ docker/postgres/backup_postgres.sh | 3 +- docker/postgres/postgresql-13.conf.j2 | 4 +-- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml index 1c7f6773e..ff016097a 100644 --- a/ansible/roles/postgres/tasks/main.yml +++ b/ansible/roles/postgres/tasks/main.yml @@ -20,21 +20,16 @@ path: /srv/newsblur/backups state: directory mode: 0777 - -- name: Start postgres docker containers + +- name: Start postgres basebackup on secondary become: yes docker_container: name: postgres image: postgres:13 state: started container_default_behavior: no_defaults - command: postgres -c config_file=/etc/postgresql/postgresql.conf - env: - # POSTGRES_USER: "{{ postgres_user }}" # Don't auto-create newsblur, manually add it - POSTGRES_PASSWORD: "{{ postgres_password }}" - hostname: "{{ inventory_hostname }}" + command: pg_basebackup -h db-postgres.service.nyc1.consul -p 5432 -U newsblur -D /var/lib/postgresql/main -Fp -R -Xs -P -c fast networks_cli_compatible: yes - # network_mode: host network_mode: default networks: - name: newsblurnet @@ -48,6 +43,35 @@ - /srv/newsblur/docker/postgres/postgres_hba-13.conf:/etc/postgresql/pg_hba.conf - /srv/newsblur/backups/:/var/lib/postgresql/backup/ restart_policy: unless-stopped + when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-postgres-secondary'] + +- name: Start postgres docker containers + become: yes + docker_container: + name: postgres + image: postgres:13 + state: started + container_default_behavior: no_defaults + command: postgres -c config_file=/etc/postgresql/postgresql.conf + env: + # POSTGRES_USER: "{{ postgres_user }}" # Don't auto-create newsblur, manually add it + POSTGRES_PASSWORD: "{{ postgres_password }}" + hostname: "{{ inventory_hostname }}" + networks_cli_compatible: yes + network_mode: default + networks: + - name: newsblurnet + aliases: + - postgres + ports: + - 5432:5432 + volumes: + - /srv/newsblur/docker/volumes/postgres:/var/lib/postgresql + - /srv/newsblur/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf + - /srv/newsblur/docker/postgres/postgres_hba-13.conf:/etc/postgresql/pg_hba.conf + - /srv/newsblur/backups/:/var/lib/postgresql/backup/ + restart_policy: unless-stopped + when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-postgres-primary', 'db-postgres'] - name: Ensure newsblur role in postgres shell: > diff --git a/docker/postgres/backup_postgres.sh b/docker/postgres/backup_postgres.sh index df58d450d..2e51660b0 100755 --- a/docker/postgres/backup_postgres.sh +++ b/docker/postgres/backup_postgres.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash now=$(date '+%Y-%m-%d-%H-%M') +BACKUP_FILENAME="backup_postgresql_${now}.sql" BACKUP_PATH="/var/lib/postgresql/backup/" UPLOAD_PATH="/srv/newsblur/backups/" -BACKUP_FILENAME="backup_postgresql_${now}.sql" BACKUP_FILE="${BACKUP_PATH}${BACKUP_FILENAME}" UPLOAD_FILE="${UPLOAD_PATH}${BACKUP_FILENAME}" + echo $(date -u) "---> PG dumping - ${now}: ${BACKUP_FILE}" sudo docker exec postgres sh -c "mkdir -p $BACKUP_PATH" sudo docker exec postgres sh -c "/usr/lib/postgresql/13/bin/pg_dump -U newsblur -h 127.0.0.1 -Fc newsblur > $BACKUP_FILE" diff --git a/docker/postgres/postgresql-13.conf.j2 b/docker/postgres/postgresql-13.conf.j2 index 60309c088..5a965dbaf 100644 --- a/docker/postgres/postgresql-13.conf.j2 +++ b/docker/postgres/postgresql-13.conf.j2 @@ -317,10 +317,10 @@ restore_command = 'cp /var/lib/postgresql/archive/%f %p' # command to use to re #promote_trigger_file = '' # file name whose presence ends recovery hot_standby = on # "off" disallows queries during recovery # (change requires restart) -#max_standby_archive_delay = 30s # max delay before canceling queries +max_standby_archive_delay = 900s # max delay before canceling queries # when reading WAL from archive; # -1 allows indefinite delay -#max_standby_streaming_delay = 30s # max delay before canceling queries +max_standby_streaming_delay = 900s # max delay before canceling queries # when reading streaming WAL; # -1 allows indefinite delay #wal_receiver_create_temp_slot = off # create temp slot if primary_slot_name From b586d519c9ddbbe7db5f1090f757bb3791212e40 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 2 May 2022 09:32:35 -0400 Subject: [PATCH 7/7] Moving pg basebackup to a backup role --- ansible/roles/backups/tasks/main.yml | 12 ++++- ansible/roles/postgres/tasks/main.yml | 55 +++++++------------ docker/postgres/backup_postgres.sh | 4 +- docker/postgres/postgresql-13.conf.j2 | 2 +- terraform/digitalocean.tf | 78 +++++++++++++-------------- 5 files changed, 72 insertions(+), 79 deletions(-) diff --git a/ansible/roles/backups/tasks/main.yml b/ansible/roles/backups/tasks/main.yml index 00bff2dfb..a6da272b7 100644 --- a/ansible/roles/backups/tasks/main.yml +++ b/ansible/roles/backups/tasks/main.yml @@ -41,7 +41,7 @@ # - dir: /backup_redis/ # file: "{{ redis_filename }}" tags: never, restore_postgres, restore_mongo, restore_redis, restore_redis_story - + - name: Restore postgres block: - name: pg_restore @@ -76,3 +76,13 @@ command: "mv -f /srv/newsblur/backups/{{ redis_story_filename }} /srv/newsblur/docker/volumes/redis/dump.rdb" ignore_errors: yes tags: never, restore_redis_story + + - name: Start postgres basebackup on secondary + become: yes + command: + docker exec postgres pg_basebackup -h db-postgres.service.nyc1.consul -p 5432 -U newsblur -D /var/lib/postgresql/data -Fp -R -Xs -P -c fast + # when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-postgres-secondary'] + tags: + - never + - restore_pg_basebackup + diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml index ff016097a..9704c0a1e 100644 --- a/ansible/roles/postgres/tasks/main.yml +++ b/ansible/roles/postgres/tasks/main.yml @@ -8,45 +8,24 @@ register: updated_config - name: Ensure postgres archive directory - become: yes file: path: /srv/newsblur/docker/volumes/postgres/archive state: directory - mode: 0777 + mode: 0755 - name: Ensure postgres backup directory - become: yes file: - path: /srv/newsblur/backups + path: /srv/newsblur/docker/volumes/postgres/backups state: directory - mode: 0777 - -- name: Start postgres basebackup on secondary - become: yes - docker_container: - name: postgres - image: postgres:13 - state: started - container_default_behavior: no_defaults - command: pg_basebackup -h db-postgres.service.nyc1.consul -p 5432 -U newsblur -D /var/lib/postgresql/main -Fp -R -Xs -P -c fast - networks_cli_compatible: yes - network_mode: default - networks: - - name: newsblurnet - aliases: - - postgres - ports: - - 5432:5432 - volumes: - - /srv/newsblur/docker/volumes/postgres:/var/lib/postgresql - - /srv/newsblur/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf - - /srv/newsblur/docker/postgres/postgres_hba-13.conf:/etc/postgresql/pg_hba.conf - - /srv/newsblur/backups/:/var/lib/postgresql/backup/ - restart_policy: unless-stopped - when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-postgres-secondary'] + mode: 0755 + +- name: Ensure postgres data directory + file: + path: /srv/newsblur/docker/volumes/postgres/data + state: directory + mode: 0755 - name: Start postgres docker containers - become: yes docker_container: name: postgres image: postgres:13 @@ -66,16 +45,21 @@ ports: - 5432:5432 volumes: - - /srv/newsblur/docker/volumes/postgres:/var/lib/postgresql + - /srv/newsblur/docker/volumes/postgres/data:/var/lib/postgresql/data + - /srv/newsblur/docker/volumes/postgres/archive:/var/lib/postgresql/archive + - /srv/newsblur/docker/volumes/postgres/backups:/var/lib/postgresql/backups - /srv/newsblur/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf - /srv/newsblur/docker/postgres/postgres_hba-13.conf:/etc/postgresql/pg_hba.conf - - /srv/newsblur/backups/:/var/lib/postgresql/backup/ restart_policy: unless-stopped when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-postgres-primary', 'db-postgres'] +- name: Change ownership in postgres docker container + command: > + docker exec postgres chown -fR postgres.postgres /var/lib/postgresql + - name: Ensure newsblur role in postgres shell: > - sleep 5; + sleep 15; docker exec postgres createuser -s newsblur -U postgres; docker exec postgres createdb newsblur -U newsblur; register: ensure_role @@ -101,11 +85,10 @@ register: app_changed - name: Add sanity checkers cronjob for disk usage - become: yes cron: name: disk_usage_sanity_checker - user: root - cron_file: /etc/cron.hourly/disk_usage_sanity_checker + minute: "0" + hour: "0" job: >- docker pull newsblur/newsblur_python3:latest; docker run --rm -it diff --git a/docker/postgres/backup_postgres.sh b/docker/postgres/backup_postgres.sh index 2e51660b0..2ba5e5217 100755 --- a/docker/postgres/backup_postgres.sh +++ b/docker/postgres/backup_postgres.sh @@ -2,8 +2,8 @@ now=$(date '+%Y-%m-%d-%H-%M') BACKUP_FILENAME="backup_postgresql_${now}.sql" -BACKUP_PATH="/var/lib/postgresql/backup/" -UPLOAD_PATH="/srv/newsblur/backups/" +BACKUP_PATH="/var/lib/postgresql/backups/" +UPLOAD_PATH="/srv/newsblur/docker/volumes/postgres/backups/" BACKUP_FILE="${BACKUP_PATH}${BACKUP_FILENAME}" UPLOAD_FILE="${UPLOAD_PATH}${BACKUP_FILENAME}" diff --git a/docker/postgres/postgresql-13.conf.j2 b/docker/postgres/postgresql-13.conf.j2 index 5a965dbaf..46ef56aa6 100644 --- a/docker/postgres/postgresql-13.conf.j2 +++ b/docker/postgres/postgresql-13.conf.j2 @@ -38,7 +38,7 @@ # The default values of these variables are driven from the -D command-line # option or PGDATA environment variable, represented here as ConfigDir. -data_directory = '/var/lib/postgresql/main' # use data in another directory +data_directory = '/var/lib/postgresql/data' # use data in another directory # (change requires restart) hba_file = '/etc/postgresql/pg_hba.conf' # host-based authentication file # (change requires restart) diff --git a/terraform/digitalocean.tf b/terraform/digitalocean.tf index 3978001cd..08ff6d2e6 100644 --- a/terraform/digitalocean.tf +++ b/terraform/digitalocean.tf @@ -23,45 +23,44 @@ resource "digitalocean_ssh_key" "default" { public_key = file("/srv/secrets-newsblur/keys/docker.key.pub") } -resource "digitalocean_project" "NewsBlur_Docker" { - name = "NewsBlur Docker" - environment = "Production" - description = "Infrastructure glued together with consul" -} +# resource "digitalocean_project" "NewsBlur_Docker" { +# name = "NewsBlur Docker" +# environment = "Production" +# description = "Infrastructure glued together with consul" +# } -resource "digitalocean_project_resources" "NewsBlur_Docker" { - project = digitalocean_project.NewsBlur_Docker.id - resources = flatten([ - digitalocean_droplet.db-consul.*.urn, - digitalocean_droplet.www.*.urn, - digitalocean_droplet.app-django.*.urn, - digitalocean_droplet.app-counts.*.urn, - digitalocean_droplet.app-push.*.urn, - digitalocean_droplet.app-refresh.*.urn, - digitalocean_droplet.blog.*.urn, - digitalocean_droplet.staging-web.*.urn, - digitalocean_droplet.discovery.*.urn, - digitalocean_droplet.node-text.*.urn, - digitalocean_droplet.node-socket.*.urn, - digitalocean_droplet.node-favicons.*.urn, - digitalocean_droplet.node-images.*.urn, - digitalocean_droplet.node-page.*.urn, - digitalocean_droplet.db-elasticsearch.*.urn, - digitalocean_droplet.db-redis-user.*.urn, - digitalocean_droplet.db-redis-sessions.*.urn, - digitalocean_droplet.db-redis-story.*.urn, - digitalocean_droplet.db-redis-pubsub.*.urn, - digitalocean_droplet.db-postgres.*.urn, - digitalocean_droplet.db-mongo-primary.*.urn, - digitalocean_droplet.db-mongo-secondary.*.urn, - digitalocean_droplet.db-mongo-analytics.*.urn, - digitalocean_droplet.db-metrics.*.urn, - digitalocean_droplet.db-sentry.*.urn, - digitalocean_droplet.task-celery.*.urn, - digitalocean_droplet.task-work.*.urn - ]) - -} +# resource "digitalocean_project_resources" "NewsBlur_Docker" { +# project = digitalocean_project.NewsBlur_Docker.id +# resources = flatten([ +# digitalocean_droplet.db-consul.*.urn, +# digitalocean_droplet.www.*.urn, +# digitalocean_droplet.app-django.*.urn, +# digitalocean_droplet.app-counts.*.urn, +# digitalocean_droplet.app-push.*.urn, +# digitalocean_droplet.app-refresh.*.urn, +# digitalocean_droplet.blog.*.urn, +# digitalocean_droplet.staging-web.*.urn, +# digitalocean_droplet.discovery.*.urn, +# digitalocean_droplet.node-text.*.urn, +# digitalocean_droplet.node-socket.*.urn, +# digitalocean_droplet.node-favicons.*.urn, +# digitalocean_droplet.node-images.*.urn, +# digitalocean_droplet.node-page.*.urn, +# digitalocean_droplet.db-elasticsearch.*.urn, +# digitalocean_droplet.db-redis-user.*.urn, +# digitalocean_droplet.db-redis-sessions.*.urn, +# digitalocean_droplet.db-redis-story.*.urn, +# digitalocean_droplet.db-redis-pubsub.*.urn, +# digitalocean_droplet.db-postgres.*.urn, +# digitalocean_droplet.db-mongo-primary.*.urn, +# digitalocean_droplet.db-mongo-secondary.*.urn, +# digitalocean_droplet.db-mongo-analytics.*.urn, +# digitalocean_droplet.db-metrics.*.urn, +# digitalocean_droplet.db-sentry.*.urn, +# digitalocean_droplet.task-celery.*.urn, +# digitalocean_droplet.task-work.*.urn +# ]) +# } # ################# # # Resources # @@ -411,8 +410,9 @@ resource "digitalocean_droplet" "db-redis-pubsub" { } resource "digitalocean_droplet" "db-postgres" { + count = 2 image = var.droplet_os - name = "db-postgres" + name = "db-postgres${count.index+1}" region = var.droplet_region size = var.droplet_size_160 ssh_keys = [digitalocean_ssh_key.default.fingerprint]