Merge branch 'master' into mongoauth

* master:
  8 app-refresh servers
  Adding volume for metrics server.
  Adding imageproxy2 before switching image proxies.
  Ignoring symlink requirements.txt.

.vscode/settings.json

@@ -21,5 +21,6 @@
         "static/*.js": true,
         "blog/.jekyll-cache": true,
         "blog/_site": true,
+        "requirements.txt": true, // It's just a symlink to config/requirements.txt, which has git history
     },
 }

ansible/roles/node/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 - name: Copy node secrets
   copy:
-    src: /srv/secrets-newsblur/settings/dotenv.env
+    src: /srv/secrets-newsblur/settings/node_settings.env
     dest: /srv/newsblur/node/.env
   register: app_changed
   notify: restart node
@@ -108,4 +108,4 @@
       -v /srv/newsblur:/srv/newsblur
       --network=newsblurnet
       --hostname {{ ansible_hostname }} 
-      newsblur/newsblur_python3 /srv/newsblur/utils/monitor_disk_usage.py $OUTPUT
+      newsblur/newsblur_python3 /srv/newsblur/utils/monitor_disk_usage.py $OUTPUT

docker/haproxy/haproxy.consul.cfg.j2

@@ -63,6 +63,7 @@ frontend public
     use_backend node_favicon if { path_beg /rss_feeds/icon/ }
     use_backend node_text if { path_beg /rss_feeds/original_text_fetcher }
     use_backend node_images if { hdr_end(host) -i imageproxy.newsblur.com }
+    use_backend node_images if { hdr_end(host) -i imageproxy2.newsblur.com }
     use_backend node_page if { path_beg /original_page/ }
     use_backend blog if { hdr_end(host) -i blog.newsblur.com }
     use_backend blog if { hdr_end(host) -i blog2.newsblur.com }

terraform/digitalocean.tf

@@ -117,7 +117,7 @@ resource "digitalocean_droplet" "app-push" {
 }
 
 resource "digitalocean_droplet" "app-refresh" {
-  count    = 2
+  count    = 8
   image    = var.droplet_os
   name     = "app-refresh${count.index+1}"
   region   = var.droplet_region
@@ -446,11 +446,22 @@ resource "digitalocean_droplet" "db-mongo-secondary" {
 #   }
 # }
 
+resource "digitalocean_volume" "metrics_volume" {
+  count                   = 0
+  region                  = "nyc1"
+  name                    = "metrics"
+  size                    = 100
+  initial_filesystem_type = "xfs"
+  description             = "Storage for NewsBlur Prometheus metrics"
+}
+
 resource "digitalocean_droplet" "db-metrics" {
   image    = var.droplet_os
   name     = "db-metrics"
   region   = var.droplet_region
-  size     = var.droplet_size
+  size     = var.metrics_droplet_size
+  # volume_ids = [digitalocean_volume.metrics_volume.0.id] 
+  volume_ids = ["f815908f-e1b7-11eb-a10f-0a58ac145428"] # 100GB volume created outside TF. Remove when upgrading to 200GB
   ssh_keys = [digitalocean_ssh_key.default.fingerprint]
   provisioner "local-exec" {
     command = "/srv/newsblur/ansible/utils/generate_inventory.py; sleep 120"

terraform/variables.tf

@@ -29,6 +29,11 @@ variable "mongo_droplet_size" {
   default = "m-4vcpu-32gb"
 }
 
+variable "metrics_droplet_size" {
+  type = string
+  default = "s-1vcpu-2gb"
+}
+
 variable "mongo_secondary_droplet_size" {
   type = string
   default = "m-2vcpu-16gb"

utils/tlnb.py

@@ -28,6 +28,8 @@ def main(roles=None, command=None, path=None):
 
     if not roles:
         roles = ['app']
+    if not isinstance(roles, list):
+        roles = [roles]
 
     while True:
         try:
@@ -50,7 +52,7 @@ def main(roles=None, command=None, path=None):
 def create_streams_for_roles(hosts, roles, command=None, path=None):
     streams = list()
     found = set()
-
+    print(path)
     if not path:
         path = "/srv/newsblur/logs/newsblur.log"
     if not command:
@@ -62,10 +64,7 @@ def create_streams_for_roles(hosts, roles, command=None, path=None):
                 follow_host(hosts, streams, found, hostname, command, path)
         else:
             host = role
-            role = re.search(r'([^0-9]+)', host).group()
-            for hostname in hosts[role]:
-                if hostname['name'] == host:
-                    follow_host(hosts, streams, found, hostname, command, path)
+            follow_host(hosts, streams, found, host, command)
 
     return streams