Merge branch 'master' into pro

* master:
  Don't change file permissions, let's see what happens.
  Bumping postgres server size, fixing monitor disk usage cronjob.

Makefile

@@ -165,6 +165,7 @@ node: deploy_node
 deploy_task:
 	- ansible-playbook ansible/deploy.yml -l task
 task: deploy_task
+celery: deploy_task
 deploy_www:
 	- ansible-playbook ansible/deploy.yml -l haproxy
 www: deploy_www

ansible/playbooks/setup_app.yml

@@ -1,7 +1,7 @@
 ---
 - name: SETUP -> app containers
   hosts: web
-  serial: "3"
+  # serial: "3"
   vars_files:
     - ../env_vars/base.yml
   vars:

ansible/roles/backups/tasks/main.yml

@@ -94,9 +94,14 @@
       - name: pg_basebackup
         become: yes
         command:
-          docker run --rm --name=pg_basebackup--network=host -e POSTGRES_PASSWORD=newsblur -v /srv/newsblur/docker/volumes/postgres/data:/var/lib/postgresql/data postgres:13 pg_basebackup -h db-postgres.service.nyc1.consul -p 5432 -U newsblur -D /var/lib/postgresql/data -Fp -R -Xs -P -c fast
+          docker run --rm --name=pg_basebackup --network=host -e POSTGRES_PASSWORD=newsblur -v /srv/newsblur/docker/volumes/postgres/data:/var/lib/postgresql/data postgres:13 pg_basebackup -h db-postgres.service.nyc1.consul -p 5432 -U newsblur -D /var/lib/postgresql/data -Fp -R -Xs -P -c fast
+      
+      - name: start postgresql
+        become: yes
+        command:
+          docker start postgres
     # when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-postgres-secondary']
     tags:
       - never
-      - restore_pg_basebackup
+      - pg_basebackup
 

ansible/roles/postgres/tasks/main.yml

@@ -95,15 +95,17 @@
   cron:
     name: disk_usage_sanity_checker
     minute: "0"
-    hour: "0"
     job: >-
-      docker pull newsblur/newsblur_python3:latest;
-      docker run --rm -it
-      OUTPUT=$(eval sudo df / | head -n 2 |  tail -1);
-      -v /srv/newsblur:/srv/newsblur
-      --network=newsblurnet
-      --hostname {{ ansible_hostname }} 
-      newsblur/newsblur_python3 /srv/newsblur/utils/monitor_disk_usage.py $OUTPUT
+      OUTPUT=$(df / | head -n 2 |  tail -1) docker run --rm -it -v /srv/newsblur:/srv/newsblur --network=newsblurnet --hostname {{ ansible_hostname }} newsblur/newsblur_python3 /srv/newsblur/utils/monitor_disk_usage.py $OUTPUT
+  tags: cron
+
+- name: Add postgresql archive cleaner cronjob
+  cron:
+    name: postgres_archive_cleaner
+    minute: "0"
+    job: >-
+      sudo find /srv/newsblur/docker/volumes/postgres/archive -type f -mmin +180 -delete
+  tags: cron
 
 - name: Add postgres backup log
   become: yes
@@ -120,4 +122,5 @@
     minute: "0"
     hour: "4"
     job: /srv/newsblur/docker/postgres/backup_postgres.sh >> /var/log/postgres_backup.log 2>&1
+  tags: cron
 

ansible/roles/postgres/templates/consul_service.json

@@ -1,6 +1,6 @@
 {
     "service": {
-        "name": "db-postgres-staging",
+        "name": "db-postgres-secondary",
         "tags": [
             "db"
         ],

ansible/roles/repo/tasks/main.yml

@@ -8,14 +8,14 @@
     owner: nb
     group: nb
 
-- name: Ensure nb /srv/newsblur owner
-  become: yes
-  file:
-    path: /srv/newsblur
-    state: directory
-    owner: nb
-    group: nb
-    recurse: yes
+# - name: Ensure nb /srv/newsblur owner
+#   become: yes
+#   file:
+#     path: /srv/newsblur
+#     state: directory
+#     owner: nb
+#     group: nb
+#     recurse: yes
 
 - name: Pull newsblur_web github
   git:

ansible/roles/web/tasks/main.yml

@@ -104,6 +104,15 @@
   tags:
     - logrotate
 
+- name: Force reload gunicorn
+  debug: 
+    msg: Forcing reload...
+  register: app_changed
+  changed_when: true
+  tags:
+    - never
+    - force
+
 - name: Reload gunicorn
   debug: 
     msg: Reloading gunicorn

docker/haproxy/haproxy.consul.cfg.j2

@@ -183,7 +183,10 @@ backend node_page
 
 backend postgres
     option httpchk GET /db_check/postgres
-    server db-postgres db-postgres.node.nyc1.consul:5579 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    {% for host in groups.postgres %}
+        server {{host}} {{host}}.node.nyc1.consul:5579
+    {% endfor %}
 
 backend mongo
     option httpchk GET /db_check/mongo

templates/maintenance_off.html

@@ -85,7 +85,7 @@
                         <img src="/media/img/logo_512.png" class="logo">
                         <h1>NewsBlur is in <span class="error404">maintenance mode</span></h1>
                         <div class="description">
-                            <p>This is a unique maintenance mode. This is only for deploying the new front-end asset packager we're switching to (<a href="https://github.com/samuelclay/NewsBlur/commit/26dd479714812b14791834f4f7854a3068e37a1c" target="window">here's the merge on GitHub</a>). It's a bit messy though, since there is a brief moment when switching from the old to the new that can be in a buggy, liminal state of using part of one system and part of another. So to avoid that I'm just going to throw up a maintenance page for a quick minute.</p>
+                            <p>This is bog standard maintenance of a PostgreSQL server. Switching from a primary to a secondary should be a hot reload and require no downtime. I'm going to force about a minute of downtime because it's way easier than coordinating the handoff between feed fetchers, app servers, writing to the old database server, ensuring all writes went to the secondary before it gets promoted, and then ensuring the old database server is taken offline so it doens't rejoin the cluster. Whew. And it probably took me longer to write this than to just do the maintenance.</p>
                             <p>To pass the time, <a href="http://mltshp.com/popular">check out what's popular on MLTSHP</a>.</p>
                         </div>
                     </div>

terraform/digitalocean.tf

@@ -414,7 +414,8 @@ resource "digitalocean_droplet" "db-postgres" {
   image    = var.droplet_os
   name     = "db-postgres${count.index+1}"
   region   = var.droplet_region
-  size     = var.droplet_size_160
+  size     = contains([0], count.index) ? var.droplet_size_160 : var.droplet_size_240
+  # size     = var.droplet_size_240
   ssh_keys = [digitalocean_ssh_key.default.fingerprint]
   provisioner "local-exec" {
     command = "/srv/newsblur/ansible/utils/generate_inventory.py; sleep 120"

terraform/variables.tf

@@ -33,6 +33,11 @@ variable "droplet_size_160" {
   default = "m-4vcpu-32gb"
 }
 
+variable "droplet_size_240" {
+  type = string
+  default = "g-8vcpu-32gb"
+}
+
 variable "droplet_size_40" {
   type = string
   default = "s-4vcpu-8gb"

utils/monitor_disk_usage.py

@@ -18,8 +18,8 @@ def main():
     hostname = socket.gethostname()
     percent = int(percent.strip('%'))
     admin_email = settings.ADMINS[0][1]
-    if True:
-    #if percent > 95:
+    # if True:
+    if percent > 90:
         requests.post(
                 "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME,
                 auth=("api", settings.MAILGUN_ACCESS_KEY),