Adding discovery and staging servers. Also adding a second node socket server.

ansible/inventories/digital_ocean.yml

@@ -15,11 +15,14 @@ groups:
   work: inventory_hostname.startswith('task-work')
 
   node: inventory_hostname.startswith('node')
+  node_socket: inventory_hostname.startswith('node-socket')
 
   # debugs: inventory_hostname.startswith('debug')
 
   task: inventory_hostname.startswith('task')
   
+  staging: inventory_hostname.startswith('staging')
+  
   db: inventory_hostname.startswith('db')
   search: inventory_hostname.startswith('db-elasticsearch')
   elasticsearch: inventory_hostname.startswith('db-elasticsearch')

ansible/provision.yml

@@ -19,3 +19,5 @@
 #   when: "'debugs' in group_names"
 - import_playbook: setup_consul_manager.yml
   when: "'consul' in group_names"
+- import_playbook: setup_staging.yml
+  when: "'staging' in group_names"

ansible/roles/consul-manager/templates/consul_manager.json.j2

@@ -1,5 +1,7 @@
 {
   "datacenter": "nyc1",
+  "data_dir": "/opt/consul",
+  "enable_debug": true,
   "log_level": "INFO",
   "enable_syslog": false,
   "advertise_addr": "{{ consul_manager_ip.stdout|trim }}",

ansible/roles/docker-compose/tasks/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Run docker-compose.yml
+  community.docker.docker_compose:
+    project_src: /srv/newsblur
+    

ansible/roles/node/tasks/main.yml

@@ -18,21 +18,41 @@
     - container_name: node_socket
       ports: 8888:8888
       command: node /srv/node/unread_counts.js
-      target_host: 'node-socket'
-    - container_name: 'original_pages'
-      ports: '3060:3060'
-      command: 'node /srv/node/original_page.js'
-      target_host: 'node-page'
-    - container_name: 'original_text'
-      ports: '4040:4040'
-      command: 'node /srv/node/original_text.js'
-      target_host: 'node-text'
-    - container_name: 'favicons'
-      ports: '3030:3030'
-      command: 'node /srv/node/favicons.js'
-      target_host: 'node-favicons'
+      target_host: node-socket
+    - container_name: original_pages
+      ports: 3060:3060
+      command: node /srv/node/original_page.js
+      target_host: node-page
+    - container_name: original_text
+      ports: 4040:4040
+      command: node /srv/node/original_text.js
+      target_host: node-text
+    - container_name: favicons
+      ports: 3030:3030
+      command: node /srv/node/favicons.js
+      target_host: node-favicons
   when: item.target_host in inventory_hostname
 
+- name: Start non-newsblur node docker containers
+  become: yes
+  docker_container:
+    name: "{{ item.container_name }}"
+    image: "{{ item.image }}"
+    state: started
+    container_default_behavior: no_defaults
+    pull: true
+    network_mode: host
+    env:
+      NODE_ENV: "production"
+    restart_policy: unless-stopped
+    volumes:
+      - /srv/newsblur/node:/srv/node
+  with_items:
+    - container_name: imageproxy
+      image: willnorris/imageproxy
+      ports: 8080:8080
+      target_host: node-images
+  when: item.target_host in inventory_hostname
 
 - name: Register nodes in consul
   tags: consul
@@ -42,15 +62,15 @@
     dest: /etc/consul.d/{{item.target_host}}.json
   with_items:
     - port: 8888
-      target_host: 'node-socket'
+      target_host: node-socket
     - port: 3060
-      target_host: 'node-page'
+      target_host: node-page
     - port: 4040
-      target_host: 'node-text'
+      target_host: node-text
     - port: 3030
-      target_host: 'node-favicons'
-    - port: 80
-      target_host: 'node-images'
+      target_host: node-favicons
+    - port: 8080
+      target_host: node-images
   notify:
     - reload consul
   when: item.target_host in inventory_hostname

ansible/roles/node/templates/consul_service.json

@@ -1,9 +1,26 @@
 {
     "service": {
         "name": "{{ item.target_host }}",
+        "id": "{{ inventory_hostname }}",
         "tags": [
             "node"
         ],
         "port": {{ item.port }}
+        {% if item.target_host in ["node-images", "node-favicons", "node-text", "node-page"] %} 
+                               ,
+        "checks": [{
+            "id": "{{inventory_hostname}}-node-ping",
+            {% if item.target_host == "node-images" %}
+            "http": "http://{{ ansible_ssh_host }}:8080/sc,sN1megONJiGNy-CCvqzVPTv-TWRhgSKhFlf61XAYESl4=/http:/samuelclay.com/static/images/2019%20-%20Cuba.jpg",
+            {% elif item.target_host == "node-favicons" %}
+            "http": "http://{{ ansible_ssh_host }}:3030/rss_feeds/icon/1",
+            {% elif item.target_host == "node-text" %}
+            "http": "http://{{ ansible_ssh_host }}:4040/rss_feeds/original_text_fetcher?test=1",
+            {% elif item.target_host == "node-page" %}
+            "http": "http://{{ ansible_ssh_host }}:3060/original_page/1?test=1",
+            {% endif %}
+            "interval": "15s"
+        }]
+        {% endif %}
     }
 }  

ansible/setup_discovery.yml

@@ -0,0 +1,17 @@
+---
+- name: SETUP -> discovery containers
+  hosts: discovery
+  vars_files:
+    - env_vars/base.yml
+  vars:
+    - update_apt_cache: yes
+    - motd_role: app
+    
+  roles:
+    - {role: 'base', tags: 'base'}
+    - {role: 'docker', tags: 'docker'}
+    - {role: 'repo', tags: 'repo'}
+    - {role: 'dnsmasq', tags: 'dnsmasq'}
+    - {role: 'consul', tags: 'consul'}
+    - {role: 'consul-client', tags: 'consul'}
+    - {role: 'docker-compose', tags: 'docker-compose'}

ansible/setup_staging.yml

@@ -0,0 +1,16 @@
+---
+- name: SETUP -> staging containers
+  hosts: staging
+  vars_files:
+    - env_vars/base.yml
+  vars:
+    - update_apt_cache: yes
+    - motd_role: app
+    
+  roles:
+    - {role: 'base', tags: 'base'}
+    - {role: 'docker', tags: 'docker'}
+    - {role: 'repo', tags: 'repo'}
+    - {role: 'dnsmasq', tags: 'dnsmasq'}
+    - {role: 'consul', tags: 'consul'}
+    - {role: 'consul-client', tags: 'consul'}

docker/haproxy/haproxy.consul.cfg.j2

@@ -108,11 +108,14 @@ backend app_push
 backend node_images
     option httpchk HEAD /sc,sN1megONJiGNy-CCvqzVPTv-TWRhgSKhFlf61XAYESl4=/http:/samuelclay.com/static/images/2019%20-%20Cuba.jpg
     http-check expect rstatus 200|301
-    server-template node-images 1 _node-images._tcp.service.nyc1.consul:80 check inter 2000ms resolvers consul resolve-opts allow-dup-ip
+    server-template node-images 1 _node-images._tcp.service.nyc1.consul:8080 check inter 2000ms resolvers consul resolve-opts allow-dup-ip
 
 backend node_socket
     balance roundrobin
-    server-template node-socket 1 _node-socket._tcp.service.nyc1.consul:8888 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    {% for host in groups.node_socket %}
+        server {{host}} {{host}}.node.nyc1.consul:8888
+    {% endfor %}
 
 backend node_favicon
     http-check expect rstatus 200|503

terraform/digitalocean.tf

@@ -133,6 +133,40 @@ resource "digitalocean_droplet" "app-refresh" {
   }
 }
 
+resource "digitalocean_droplet" "staging-web" {
+  image    = var.droplet_os
+  name     = "staging-web"
+  region   = var.droplet_region
+  size     = var.droplet_size_20
+  ssh_keys = [digitalocean_ssh_key.default.fingerprint]
+  provisioner "local-exec" {
+    command = "/srv/newsblur/ansible/utils/generate.py; sleep 120"
+  }
+  provisioner "local-exec" {
+    command = "cd ..; ansible-playbook -l ${self.name} ansible/setup_root.yml"
+  }
+  provisioner "local-exec" {
+    command = "cd ..; ansible-playbook -l ${self.name} ansible/provision.yml"
+  }
+}
+
+resource "digitalocean_droplet" "discovery" {
+  image    = var.droplet_os
+  name     = "discovery"
+  region   = var.droplet_region
+  size     = var.droplet_size_120
+  ssh_keys = [digitalocean_ssh_key.default.fingerprint]
+  provisioner "local-exec" {
+    command = "/srv/newsblur/ansible/utils/generate.py; sleep 120"
+  }
+  provisioner "local-exec" {
+    command = "cd ..; ansible-playbook -l ${self.name} ansible/setup_root.yml"
+  }
+  provisioner "local-exec" {
+    command = "cd ..; ansible-playbook -l ${self.name} ansible/provision.yml"
+  }
+}
+
 resource "digitalocean_droplet" "node-text" {
   image    = var.droplet_os
   name     = "node-text"
@@ -151,8 +185,9 @@ resource "digitalocean_droplet" "node-text" {
 }
 
 resource "digitalocean_droplet" "node-socket" {
+  count    = 2
   image    = var.droplet_os
-  name     = "node-socket"
+  name     = "node-socket${count.index+1}"
   region   = var.droplet_region
   size     = var.droplet_size
   ssh_keys = [digitalocean_ssh_key.default.fingerprint]

terraform/variables.tf

@@ -8,6 +8,16 @@ variable "droplet_size" {
   default = "s-1vcpu-1gb"
 }
 
+variable "droplet_size_20" {
+  type = string
+  default = "s-2vcpu-4gb"
+}
+
+variable "droplet_size_120" {
+  type = string
+  default = "g-4vcpu-16gb"
+}
+
 variable "droplet_os" {
   type = string
   default = "ubuntu-20-04-x64"