fix up prometheus and try to get grafana/prometheus working in ansible (Work in progress)

ansible/roles/grafana/tasks/main.yml

@@ -9,6 +9,7 @@
   docker_container:
     name: grafana
     image: grafana/grafana
+    user: root
     env:
       GF_SECURITY_ADMIN_PASSWORD: "{{ grafana_password }}"
     ports:

ansible/roles/prometheus/tasks/main.yml

@@ -1,10 +1,4 @@
 ---
-- name: Register Manager IP
-  run_once: yes
-  become: no
-  register: consul_manager_ip
-  local_action: command /srv/newsblur/ansible/roles/consul/tasks/get_consul_manager_ip.py
-
 - name: Template file for prometheus
   template:
     src: /srv/newsblur/docker/prometheus/prometheus.consul.yml.j2
@@ -14,12 +8,12 @@
   become: yes
   docker_container:
     name: prometheus
-    image: prom/prometheus:0.18.0
+    image: prom/prometheus
+    user: root
     state: started
     network_mode: host
     restart_policy: unless-stopped
-    command: '-config.file=/etc/prometheus/prometheus.yml'
-    ports:
+    expose:
       - "9090:9090"
     volumes:
       - /srv/newsblur/docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml

docker/grafana/datasources/consul.datasource.yaml

@@ -0,0 +1,7 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://db-metrics.service.nyc1.consul:9090

docker/prometheus/prometheus.consul.yml.j2

@@ -2,83 +2,82 @@ global:
     scrape_interval: 5s
 scrape_configs:
     - job_name: 'consul-services'
-        - targets: ['localhost:]
       consul_sd_configs:
-      - server: '{{ consul_manager_ip }}:8500'
-      services: [node-exporter]
+      - server: 'db-consul.service.nyc1.consul:8500'
+        services: [node-exporter]
   
     - job_name: 'app_servers'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/app-servers
       scheme: https
     - job_name: 'app_times'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/app-times
       scheme: https
     - job_name: 'classifiers'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/classifiers
       scheme: https
     - job_name: 'db_times'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/db-times
       scheme: https
     - job_name: 'errors'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/errors
       scheme: https
     - job_name: 'feed_counts'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/feed-counts
       scheme: https
     - job_name: 'feeds'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/feeds
       scheme: https
     - job_name: 'load_times'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/load-times
       scheme: https
     - job_name: 'stories'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/stories
       scheme: https
     - job_name: 'task_codes'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/task-codes
       scheme: https
     - job_name: 'task_pipeline'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/task-pipeline
       scheme: https
     - job_name: 'task_servers'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/task-servers
       scheme: https
     - job_name: 'task_times'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/task-times
       scheme: https
     - job_name: 'updates'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/updates
       scheme: https
     - job_name: 'users'
-      target_groups:
+      static_configs:
         - targets: ['haproxy.service.nyc1.consul']
       metrics_path: /monitor/users
       scheme: https