diff --git a/ansible/playbooks/setup_metrics.yml b/ansible/playbooks/setup_metrics.yml index e198a662e..15f574383 100644 --- a/ansible/playbooks/setup_metrics.yml +++ b/ansible/playbooks/setup_metrics.yml @@ -15,7 +15,9 @@ - {role: 'dnsmasq', tags: 'dnsmasq'} - {role: 'consul', tags: 'consul'} - {role: 'consul-client', tags: 'consul'} - + - {role: 'mongo-exporter', tags: 'mongo-exporter'} + - {role: 'postgres-exporter', tags: 'postgres-exporter'} + - {role: 'redis-exporter', tags: 'redis-exporter'} - {role: 'node-exporter', tags: ['node-exporter', 'metrics']} - {role: 'prometheus', tags: ['prometheus', 'metrics']} - {role: 'grafana', tags: ['grafana', 'metrics']} diff --git a/ansible/roles/mongo-exporter/tasks/main.yml b/ansible/roles/mongo-exporter/tasks/main.yml new file mode 100644 index 000000000..9b1b07b9a --- /dev/null +++ b/ansible/roles/mongo-exporter/tasks/main.yml @@ -0,0 +1,30 @@ +- name: Make docker network for newsblurnet + become: yes + docker_network: + name: newsblurnet + notify: restart docker + +- name: Start mongo-exporter container + become: yes + docker_container: + name: mongo-exporter + image: bitnami/mongodb-exporter + restart_policy: unless-stopped + container_default_behavior: no_defaults + networks_cli_compatible: yes + network_mode: default + networks: + - name: newsblurnet + env: + MONGODB_URI: 'mongodb://db-mongo.service.nyc1.consul:27017/admin?' + ports: + - '9216:9216' + +- name: Register mongo-exporter in consul + tags: consul + become: yes + template: + src: consul_service.json + dest: /etc/consul.d/mongo-exporter.json + notify: + - reload consul \ No newline at end of file diff --git a/ansible/roles/mongo-exporter/templates/consul_service.json b/ansible/roles/mongo-exporter/templates/consul_service.json new file mode 100644 index 000000000..7ac4bf233 --- /dev/null +++ b/ansible/roles/mongo-exporter/templates/consul_service.json @@ -0,0 +1,18 @@ +{ + "service": { + "name": "mongo-exporter", + "id": "{{ inventory_hostname }}-exporter", + "tags": [ + "mongo-exporter" + ], + "port": 9216, + "checks": [ + { + "id": "{{inventory_hostname}}-exporter-ping", + "http": "http://{{ ansible_ssh_host }}:9216", + "interval": "15s", + "failures_before_critical": 4 + } + ] + } +} \ No newline at end of file diff --git a/ansible/roles/postgres-exporter/tasks/get_postgres_credentials.py b/ansible/roles/postgres-exporter/tasks/get_postgres_credentials.py new file mode 100755 index 000000000..d93df3e08 --- /dev/null +++ b/ansible/roles/postgres-exporter/tasks/get_postgres_credentials.py @@ -0,0 +1,8 @@ +#!/srv/newsblur/venv/newsblur3/bin/python +import sys +sys.path.append('/srv/newsblur') +from newsblur_web import settings + +username = settings.DATABASES['default']['USER'] +password = settings.DATABASES['default']['PASSWORD'] +print(f"{username}:{password}") \ No newline at end of file diff --git a/ansible/roles/postgres-exporter/tasks/main.yml b/ansible/roles/postgres-exporter/tasks/main.yml new file mode 100644 index 000000000..0da77b2aa --- /dev/null +++ b/ansible/roles/postgres-exporter/tasks/main.yml @@ -0,0 +1,35 @@ +- name: Make docker network for newsblurnet + become: yes + docker_network: + name: newsblurnet + notify: restart docker + +- name: Register Postgres user and password + become: no + run_once: yes + register: postgres_credentials + local_action: command /srv/newsblur/ansible/roles/postgres-exporter/tasks/get_postgres_credentials.py +- name: Start postgres-exporter container + become: yes + docker_container: + name: postgres-exporter + image: prometheuscommunity/postgres-exporter + restart_policy: unless-stopped + container_default_behavior: no_defaults + networks_cli_compatible: yes + network_mode: default + networks: + - name: newsblurnet + env: + DATA_SOURCE_NAME: 'postgresql://{{ postgres_credentials }}@db-postgres.service.nyc1.consul:5432/postgres' + ports: + - '9187:9187' + +- name: Register postgres-exporter in consul + tags: consul + become: yes + template: + src: consul_service.json + dest: /etc/consul.d/postgres-exporter.json + notify: + - reload consul \ No newline at end of file diff --git a/ansible/roles/postgres-exporter/templates/consul_service.json b/ansible/roles/postgres-exporter/templates/consul_service.json new file mode 100644 index 000000000..1e6e87162 --- /dev/null +++ b/ansible/roles/postgres-exporter/templates/consul_service.json @@ -0,0 +1,18 @@ +{ + "service": { + "name": "postgres-exporter", + "id": "{{ inventory_hostname }}-exporter", + "tags": [ + "postgres-exporter" + ], + "port": 9187, + "checks": [ + { + "id": "{{inventory_hostname}}-exporter-ping", + "http": "http://{{ ansible_ssh_host }}:9187", + "interval": "15s", + "failures_before_critical": 4 + } + ] + } +} \ No newline at end of file diff --git a/ansible/roles/redis-exporter/tasks/main.yml b/ansible/roles/redis-exporter/tasks/main.yml new file mode 100644 index 000000000..4b29c806d --- /dev/null +++ b/ansible/roles/redis-exporter/tasks/main.yml @@ -0,0 +1,48 @@ +- name: Make docker network for newsblurnet + become: yes + docker_network: + name: newsblurnet + notify: restart docker + +- name: Start redis-exporter containers + become: yes + docker_container: + name: "{{item.redis_target}}-exporter" + image: oliver006/redis_exporter + restart_policy: unless-stopped + container_default_behavior: no_defaults + env: + REDIS_ADDR: "db-{{item.redis_target}}.service.nyc1.consul:6379" + networks_cli_compatible: yes + network_mode: default + networks: + - name: newsblurnet + ports: + - '{{item.port}}:9121' + with_items: + - port: 9121 + redis_target: "redis-user" + - port: 9122 + redis_target: "redis-sessions" + - port: 9123 + redis_target: "redis-story" + - port: 9124 + redis_target: "redis-pubsub" + +- name: Register redis-exporters in consul + tags: consul + become: yes + template: + src: consul_service.json + dest: /etc/consul.d/{{item.redis_target}}-exporter.json + notify: + - reload consul + with_items: + - port: 9121 + redis_target: "redis-user" + - port: 9122 + redis_target: "redis-sessions" + - port: 9123 + redis_target: "redis-story" + - port: 9124 + redis_target: "redis-pubsub" diff --git a/ansible/roles/redis-exporter/templates/consul_service.json b/ansible/roles/redis-exporter/templates/consul_service.json new file mode 100644 index 000000000..4ec5e3536 --- /dev/null +++ b/ansible/roles/redis-exporter/templates/consul_service.json @@ -0,0 +1,18 @@ +{ + "service": { + "name": "redis-exporter", + "id": "{{ item.redis_target }}-exporter", + "tags": [ + "redis-exporter" + ], + "port": {{ item.port }}, + "checks": [ + { + "id": "{{ item.redis_target }}-exporter-ping", + "http": "http://{{ ansible_ssh_host }}:{{ item.port }}", + "interval": "15s", + "failures_before_critical": 4 + } + ] + } +} \ No newline at end of file diff --git a/docker/grafana/datasources/datasource.yaml b/docker/grafana/datasources/datasource.yaml index d96153aa0..9df690950 100644 --- a/docker/grafana/datasources/datasource.yaml +++ b/docker/grafana/datasources/datasource.yaml @@ -4,4 +4,4 @@ datasources: - name: Prometheus type: prometheus access: proxy - url: http://prometheus:9090 + url: http://db-metrics.service.nyc1.consul:9090 diff --git a/docker/prometheus/prometheus.consul.yml.j2 b/docker/prometheus/prometheus.consul.yml.j2 index 6ad203eaf..d0c541a29 100644 --- a/docker/prometheus/prometheus.consul.yml.j2 +++ b/docker/prometheus/prometheus.consul.yml.j2 @@ -9,6 +9,34 @@ scrape_configs: - source_labels: ['__meta_consul_node'] target_label: instance + + - job_name: 'mongo_exporter' + consul_sd_configs: + - server: 'consul.newsblur.com:8500' + services: ['mongo-exporter'] + relabel_configs: + - source_labels: ['__meta_consul_node'] + target_label: instance + + + - job_name: 'postgres_exporter' + consul_sd_configs: + - server: 'consul.newsblur.com:8500' + services: ['postgres-exporter'] + relabel_configs: + - source_labels: ['__meta_consul_node'] + target_label: instance + + + ## config for the multiple Redis targets that the exporter will scrape + - job_name: 'redis_exporter' + consul_sd_configs: + - server: 'consul.newsblur.com:8500' + services: ['redis-exporter'] + relabel_configs: + - source_labels: ['__meta_consul_service_id'] + target_label: instance + - job_name: 'prometheus' static_configs: - targets: ['localhost:9090']