add exporters for dbs

ansible/playbooks/setup_metrics.yml

@@ -15,7 +15,9 @@
     - {role: 'dnsmasq', tags: 'dnsmasq'}
     - {role: 'consul', tags: 'consul'}
     - {role: 'consul-client', tags: 'consul'}
-
+    - {role: 'mongo-exporter', tags: 'mongo-exporter'}
+    - {role: 'postgres-exporter', tags: 'postgres-exporter'}
+    - {role: 'redis-exporter', tags: 'redis-exporter'}
     - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
     - {role: 'prometheus', tags: ['prometheus', 'metrics']}
     - {role: 'grafana', tags: ['grafana', 'metrics']}

ansible/roles/mongo-exporter/tasks/main.yml

@@ -0,0 +1,30 @@
+- name: Make docker network for newsblurnet
+  become: yes
+  docker_network:
+    name: newsblurnet
+  notify: restart docker
+
+- name: Start mongo-exporter container
+  become: yes
+  docker_container:
+    name: mongo-exporter
+    image: bitnami/mongodb-exporter
+    restart_policy: unless-stopped
+    container_default_behavior: no_defaults
+    networks_cli_compatible: yes
+    network_mode: default
+    networks:
+      - name: newsblurnet
+    env:
+      MONGODB_URI: 'mongodb://db-mongo.service.nyc1.consul:27017/admin?'
+    ports:
+    - '9216:9216'
+
+- name: Register mongo-exporter in consul
+  tags: consul
+  become: yes
+  template:
+    src: consul_service.json
+    dest: /etc/consul.d/mongo-exporter.json
+  notify:
+    - reload consul

ansible/roles/mongo-exporter/templates/consul_service.json

@@ -0,0 +1,18 @@
+{
+    "service": {
+        "name": "mongo-exporter",
+        "id": "{{ inventory_hostname }}-exporter",
+        "tags": [
+            "mongo-exporter"
+        ],
+        "port": 9216,
+        "checks": [
+            {
+                "id": "{{inventory_hostname}}-exporter-ping",
+                "http": "http://{{ ansible_ssh_host }}:9216",
+                "interval": "15s",
+                "failures_before_critical": 4
+            }
+        ]
+    }
+}

ansible/roles/postgres-exporter/tasks/get_postgres_credentials.py

@@ -0,0 +1,8 @@
+#!/srv/newsblur/venv/newsblur3/bin/python
+import sys
+sys.path.append('/srv/newsblur')
+from newsblur_web import settings
+
+username = settings.DATABASES['default']['USER']
+password = settings.DATABASES['default']['PASSWORD']
+print(f"{username}:{password}")

ansible/roles/postgres-exporter/tasks/main.yml

@@ -0,0 +1,35 @@
+- name: Make docker network for newsblurnet
+  become: yes
+  docker_network:
+    name: newsblurnet
+  notify: restart docker
+
+- name: Register Postgres user and password
+  become: no
+  run_once: yes
+  register: postgres_credentials
+  local_action: command /srv/newsblur/ansible/roles/postgres-exporter/tasks/get_postgres_credentials.py
+- name: Start postgres-exporter container
+  become: yes
+  docker_container:
+    name: postgres-exporter
+    image:  prometheuscommunity/postgres-exporter
+    restart_policy: unless-stopped
+    container_default_behavior: no_defaults
+    networks_cli_compatible: yes
+    network_mode: default
+    networks:
+      - name: newsblurnet
+    env:
+      DATA_SOURCE_NAME: 'postgresql://{{ postgres_credentials }}@db-postgres.service.nyc1.consul:5432/postgres'
+    ports:
+    - '9187:9187'
+
+- name: Register postgres-exporter in consul
+  tags: consul
+  become: yes
+  template:
+    src: consul_service.json
+    dest: /etc/consul.d/postgres-exporter.json
+  notify:
+    - reload consul

ansible/roles/postgres-exporter/templates/consul_service.json

@@ -0,0 +1,18 @@
+{
+    "service": {
+        "name": "postgres-exporter",
+        "id": "{{ inventory_hostname }}-exporter",
+        "tags": [
+            "postgres-exporter"
+        ],
+        "port": 9187,
+        "checks": [
+            {
+                "id": "{{inventory_hostname}}-exporter-ping",
+                "http": "http://{{ ansible_ssh_host }}:9187",
+                "interval": "15s",
+                "failures_before_critical": 4
+            }
+        ]
+    }
+}

ansible/roles/redis-exporter/tasks/main.yml

@@ -0,0 +1,48 @@
+- name: Make docker network for newsblurnet
+  become: yes
+  docker_network:
+    name: newsblurnet
+  notify: restart docker
+
+- name: Start redis-exporter containers
+  become: yes
+  docker_container:
+    name: "{{item.redis_target}}-exporter"
+    image: oliver006/redis_exporter
+    restart_policy: unless-stopped
+    container_default_behavior: no_defaults
+    env:
+      REDIS_ADDR: "db-{{item.redis_target}}.service.nyc1.consul:6379"
+    networks_cli_compatible: yes
+    network_mode: default
+    networks:
+      - name: newsblurnet
+    ports:
+    - '{{item.port}}:9121'
+  with_items:
+    - port: 9121
+      redis_target: "redis-user"
+    - port: 9122
+      redis_target: "redis-sessions"
+    - port: 9123
+      redis_target: "redis-story"
+    - port: 9124
+      redis_target: "redis-pubsub"
+
+- name: Register redis-exporters in consul
+  tags: consul
+  become: yes
+  template:
+    src: consul_service.json
+    dest: /etc/consul.d/{{item.redis_target}}-exporter.json
+  notify:
+    - reload consul
+  with_items:
+    - port: 9121
+      redis_target: "redis-user"
+    - port: 9122
+      redis_target: "redis-sessions"
+    - port: 9123
+      redis_target: "redis-story"
+    - port: 9124
+      redis_target: "redis-pubsub"

ansible/roles/redis-exporter/templates/consul_service.json

@@ -0,0 +1,18 @@
+{
+    "service": {
+        "name": "redis-exporter",
+        "id": "{{ item.redis_target }}-exporter",
+        "tags": [
+            "redis-exporter"
+        ],
+        "port": {{ item.port }},
+        "checks": [
+            {
+                "id": "{{ item.redis_target }}-exporter-ping",
+                "http": "http://{{ ansible_ssh_host }}:{{ item.port }}",
+                "interval": "15s",
+                "failures_before_critical": 4
+            }
+        ]
+    }
+}

docker/grafana/datasources/datasource.yaml

@@ -4,4 +4,4 @@ datasources:
   - name: Prometheus
     type: prometheus
     access: proxy
-    url: http://prometheus:9090
+    url: http://db-metrics.service.nyc1.consul:9090

docker/prometheus/prometheus.consul.yml.j2

@@ -9,6 +9,34 @@ scrape_configs:
       - source_labels: ['__meta_consul_node']
         target_label: instance
     
+
+    - job_name: 'mongo_exporter'
+      consul_sd_configs:
+      - server: 'consul.newsblur.com:8500'
+        services: ['mongo-exporter']
+      relabel_configs:
+      - source_labels: ['__meta_consul_node']
+        target_label: instance
+
+
+    - job_name: 'postgres_exporter'
+      consul_sd_configs:
+      - server: 'consul.newsblur.com:8500'
+        services: ['postgres-exporter']
+      relabel_configs:
+      - source_labels: ['__meta_consul_node']
+        target_label: instance
+
+
+    ## config for the multiple Redis targets that the exporter will scrape
+    - job_name: 'redis_exporter'
+      consul_sd_configs:
+      - server: 'consul.newsblur.com:8500'
+        services: ['redis-exporter']
+      relabel_configs:
+      - source_labels: ['__meta_consul_service_id']
+        target_label: instance
+
     - job_name: 'prometheus'
       static_configs:
         - targets: ['localhost:9090']