diff --git a/Makefile b/Makefile index 27d16deec..283f52dab 100644 --- a/Makefile +++ b/Makefile @@ -165,6 +165,7 @@ node: deploy_node deploy_task: - ansible-playbook ansible/deploy.yml -l task task: deploy_task +celery: deploy_task deploy_www: - ansible-playbook ansible/deploy.yml -l haproxy www: deploy_www diff --git a/ansible/playbooks/setup_app.yml b/ansible/playbooks/setup_app.yml index 0ce045c63..706d219b4 100644 --- a/ansible/playbooks/setup_app.yml +++ b/ansible/playbooks/setup_app.yml @@ -1,7 +1,7 @@ --- - name: SETUP -> app containers hosts: web - serial: "3" + # serial: "3" vars_files: - ../env_vars/base.yml vars: diff --git a/ansible/roles/backups/tasks/main.yml b/ansible/roles/backups/tasks/main.yml index c0e73cf53..b21ed27ab 100644 --- a/ansible/roles/backups/tasks/main.yml +++ b/ansible/roles/backups/tasks/main.yml @@ -94,9 +94,14 @@ - name: pg_basebackup become: yes command: - docker run --rm --name=pg_basebackup--network=host -e POSTGRES_PASSWORD=newsblur -v /srv/newsblur/docker/volumes/postgres/data:/var/lib/postgresql/data postgres:13 pg_basebackup -h db-postgres.service.nyc1.consul -p 5432 -U newsblur -D /var/lib/postgresql/data -Fp -R -Xs -P -c fast + docker run --rm --name=pg_basebackup --network=host -e POSTGRES_PASSWORD=newsblur -v /srv/newsblur/docker/volumes/postgres/data:/var/lib/postgresql/data postgres:13 pg_basebackup -h db-postgres.service.nyc1.consul -p 5432 -U newsblur -D /var/lib/postgresql/data -Fp -R -Xs -P -c fast + + - name: start postgresql + become: yes + command: + docker start postgres # when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-postgres-secondary'] tags: - never - - restore_pg_basebackup + - pg_basebackup diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml index 7529ee739..d17eae0a2 100644 --- a/ansible/roles/postgres/tasks/main.yml +++ b/ansible/roles/postgres/tasks/main.yml @@ -95,15 +95,17 @@ cron: name: disk_usage_sanity_checker minute: "0" - hour: "0" job: >- - docker pull newsblur/newsblur_python3:latest; - docker run --rm -it - OUTPUT=$(eval sudo df / | head -n 2 | tail -1); - -v /srv/newsblur:/srv/newsblur - --network=newsblurnet - --hostname {{ ansible_hostname }} - newsblur/newsblur_python3 /srv/newsblur/utils/monitor_disk_usage.py $OUTPUT + OUTPUT=$(df / | head -n 2 | tail -1) docker run --rm -it -v /srv/newsblur:/srv/newsblur --network=newsblurnet --hostname {{ ansible_hostname }} newsblur/newsblur_python3 /srv/newsblur/utils/monitor_disk_usage.py $OUTPUT + tags: cron + +- name: Add postgresql archive cleaner cronjob + cron: + name: postgres_archive_cleaner + minute: "0" + job: >- + sudo find /srv/newsblur/docker/volumes/postgres/archive -type f -mmin +180 -delete + tags: cron - name: Add postgres backup log become: yes @@ -120,4 +122,5 @@ minute: "0" hour: "4" job: /srv/newsblur/docker/postgres/backup_postgres.sh >> /var/log/postgres_backup.log 2>&1 + tags: cron diff --git a/ansible/roles/postgres/templates/consul_service.json b/ansible/roles/postgres/templates/consul_service.json index c416bc0c2..7e10c08f0 100644 --- a/ansible/roles/postgres/templates/consul_service.json +++ b/ansible/roles/postgres/templates/consul_service.json @@ -1,6 +1,6 @@ { "service": { - "name": "db-postgres-staging", + "name": "db-postgres-secondary", "tags": [ "db" ], diff --git a/ansible/roles/repo/tasks/main.yml b/ansible/roles/repo/tasks/main.yml index 3b45b1382..1d9dac8bc 100644 --- a/ansible/roles/repo/tasks/main.yml +++ b/ansible/roles/repo/tasks/main.yml @@ -8,14 +8,14 @@ owner: nb group: nb -- name: Ensure nb /srv/newsblur owner - become: yes - file: - path: /srv/newsblur - state: directory - owner: nb - group: nb - recurse: yes +# - name: Ensure nb /srv/newsblur owner +# become: yes +# file: +# path: /srv/newsblur +# state: directory +# owner: nb +# group: nb +# recurse: yes - name: Pull newsblur_web github git: diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml index 980c2a85f..e0252d7f4 100644 --- a/ansible/roles/web/tasks/main.yml +++ b/ansible/roles/web/tasks/main.yml @@ -104,6 +104,15 @@ tags: - logrotate +- name: Force reload gunicorn + debug: + msg: Forcing reload... + register: app_changed + changed_when: true + tags: + - never + - force + - name: Reload gunicorn debug: msg: Reloading gunicorn diff --git a/docker/haproxy/haproxy.consul.cfg.j2 b/docker/haproxy/haproxy.consul.cfg.j2 index b259968a4..ed2088a4c 100644 --- a/docker/haproxy/haproxy.consul.cfg.j2 +++ b/docker/haproxy/haproxy.consul.cfg.j2 @@ -183,7 +183,10 @@ backend node_page backend postgres option httpchk GET /db_check/postgres - server db-postgres db-postgres.node.nyc1.consul:5579 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + {% for host in groups.postgres %} + server {{host}} {{host}}.node.nyc1.consul:5579 + {% endfor %} backend mongo option httpchk GET /db_check/mongo diff --git a/templates/maintenance_off.html b/templates/maintenance_off.html index e1928a3ae..4dc4457c4 100644 --- a/templates/maintenance_off.html +++ b/templates/maintenance_off.html @@ -85,7 +85,7 @@

NewsBlur is in maintenance mode

-

This is a unique maintenance mode. This is only for deploying the new front-end asset packager we're switching to (here's the merge on GitHub). It's a bit messy though, since there is a brief moment when switching from the old to the new that can be in a buggy, liminal state of using part of one system and part of another. So to avoid that I'm just going to throw up a maintenance page for a quick minute.

+

This is bog standard maintenance of a PostgreSQL server. Switching from a primary to a secondary should be a hot reload and require no downtime. I'm going to force about a minute of downtime because it's way easier than coordinating the handoff between feed fetchers, app servers, writing to the old database server, ensuring all writes went to the secondary before it gets promoted, and then ensuring the old database server is taken offline so it doens't rejoin the cluster. Whew. And it probably took me longer to write this than to just do the maintenance.

To pass the time, check out what's popular on MLTSHP.

diff --git a/terraform/digitalocean.tf b/terraform/digitalocean.tf index 5e3c30f10..709df96bc 100644 --- a/terraform/digitalocean.tf +++ b/terraform/digitalocean.tf @@ -414,7 +414,8 @@ resource "digitalocean_droplet" "db-postgres" { image = var.droplet_os name = "db-postgres${count.index+1}" region = var.droplet_region - size = var.droplet_size_160 + size = contains([0], count.index) ? var.droplet_size_160 : var.droplet_size_240 + # size = var.droplet_size_240 ssh_keys = [digitalocean_ssh_key.default.fingerprint] provisioner "local-exec" { command = "/srv/newsblur/ansible/utils/generate_inventory.py; sleep 120" diff --git a/terraform/variables.tf b/terraform/variables.tf index 8b7c42b4d..6122c0b05 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -33,6 +33,11 @@ variable "droplet_size_160" { default = "m-4vcpu-32gb" } +variable "droplet_size_240" { + type = string + default = "g-8vcpu-32gb" +} + variable "droplet_size_40" { type = string default = "s-4vcpu-8gb" diff --git a/utils/monitor_disk_usage.py b/utils/monitor_disk_usage.py index 2c7c3730e..e32292c40 100755 --- a/utils/monitor_disk_usage.py +++ b/utils/monitor_disk_usage.py @@ -18,8 +18,8 @@ def main(): hostname = socket.gethostname() percent = int(percent.strip('%')) admin_email = settings.ADMINS[0][1] - if True: - #if percent > 95: + # if True: + if percent > 90: requests.post( "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME, auth=("api", settings.MAILGUN_ACCESS_KEY),