public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-04-13 09:42:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

redo prometheus scrapers and build the jobs for them in prometheus.yml

Browse source
This commit is contained in:
Jonathan Math 2021-04-30 13:48:48 -05:00
parent 0de1c8ecfc
commit 9cbad888c7
19 changed files with 358 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
apps/monitor/views/newsblur_app_servers.py
View file

@ -9,8 +9,14 @@ class AppServers(View):
data = dict((("%s" % s['_id'].replace('-', ''), s['feeds']) for s in self.stats))
if self.total:
data['total'] = self.total[0]['feeds']
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "app_servers"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)
@property
def stats(self):

10
apps/monitor/views/newsblur_app_times.py
View file

@ -8,7 +8,15 @@ class AppTimes(View):
def get(self, request):
servers = dict((("%s" % s['_id'], s['page_load']) for s in self.stats))
data = servers
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "app_times"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)
@property
def stats(self):

10
apps/monitor/views/newsblur_classifiers.py
View file

@ -13,5 +13,13 @@ class Classifiers(View):
'titles': MClassifierTitle.objects.count(),
}
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "classifiers"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)

9
apps/monitor/views/newsblur_dbtimes.py
View file

@ -16,5 +16,12 @@ class DbTimes(View):
'task_mongo_avg': MStatistics.get('latest_task_mongo_avg'),
'task_redis_avg': MStatistics.get('latest_task_redis_avg'),
}
chart_name = "db_times"
chart_type = "histogram"
return render(request, 'monitor/prometheus_data.html', {"data": data})
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)

10
apps/monitor/views/newsblur_errors.py
View file

@ -10,5 +10,13 @@ class Errors(View):
data = {
'feed_success': statistics['feeds_fetched'],
}
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "errors"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)

10
apps/monitor/views/newsblur_feed_counts.py
View file

@ -45,6 +45,14 @@ class FeedCounts(View):
'active_feeds': active_feeds,
'push_feeds': push_feeds,
}
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "feed_counts"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)

10
apps/monitor/views/newsblur_feeds.py
View file

@ -26,6 +26,14 @@ class Feeds(View):
'profiles': MSocialProfile.objects.count(),
'social_subscriptions': MSocialSubscription.objects.count(),
}
chart_name = "feeds"
chart_type = "histogram"
return render(request, 'monitor/prometheus_data.html', {"data": data})
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)

10
apps/monitor/views/newsblur_loadtimes.py
View file

@ -10,5 +10,13 @@ class LoadTimes(View):
'feed_loadtimes_avg_hour': MStatistics.get('latest_avg_time_taken'),
'feeds_loaded_hour': MStatistics.get('latest_sites_loaded'),
}
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "load_times"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)

10
apps/monitor/views/newsblur_stories.py
View file

@ -10,5 +10,13 @@ class Stories(View):
'stories': MStory.objects.count(),
'starred_stories': MStarredStory.objects.count(),
}
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "stories"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)

11
apps/monitor/views/newsblur_tasks_codes.py
View file

@ -7,8 +7,15 @@ class TasksCodes(View):
def get(self, request):
data = dict((("_%s" % s['_id'], s['feeds']) for s in self.stats))
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "task_codes"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)
@property
def stats(self):

10
apps/monitor/views/newsblur_tasks_pipeline.py
View file

@ -8,7 +8,15 @@ class TasksPipeline(View):
def get(self, request):
data =self.stats
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "task_pipeline"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)
@property
def stats(self):

9
apps/monitor/views/newsblur_tasks_servers.py
View file

@ -10,8 +10,15 @@ class TasksServers(View):
data = dict((("%s" % s['_id'].replace('-', ''), s['feeds']) for s in self.stats))
if self.total:
data['total'] = self.total[0]['feeds']
chart_name = "task_servers"
chart_type = "histogram"
return render(request, 'monitor/prometheus_data.html', {"data": data})
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)
@property

9
apps/monitor/views/newsblur_tasks_times.py
View file

@ -8,8 +8,15 @@ class TasksTimes(View):
def get(self, request):
data = dict((("%s" % s['_id'], s['total']) for s in self.stats))
chart_name = "task_times"
chart_type = "histogram"
return render(request, 'monitor/prometheus_data.html', {"data": data})
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)
@property

10
apps/monitor/views/newsblur_updates.py
View file

@ -20,5 +20,13 @@ class Updates(View):
'celery_work_queue': r.llen("work_queue"),
'celery_search_queue': r.llen("search_indexer"),
}
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "updates"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)

10
apps/monitor/views/newsblur_users.py
View file

@ -19,5 +19,13 @@ class Users(View):
'premium': Profile.objects.filter(is_premium=True).count(),
'queued': RNewUserQueue.user_count(),
}
return render(request, 'monitor/prometheus_data.html', {"data": data})
chart_name = "users"
chart_type = "histogram"
context = {
"data": data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context)

7
docker-compose.yml
View file

@ -110,7 +110,7 @@ services:
- 9200:9200
- 9300:9300
volumes:
- ./docker/volumes/elasticsearch:/usr/share/elasticsearch/data
- ./docker/volumes/elasticsearch:/elasticsearch/dat
- ./config/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
db_mongo:
@ -163,13 +163,18 @@ services:
- ./docker/volumes/prometheus_data:/prometheus
depends_on:
- node-exporter
- haproxy
external_links:
- haproxy
node-exporter:
container_name: node-exporter
image: prom/node-exporter
ports:
- 9100:9100
grafana:
container_name: grafana
image: grafana/grafana
environment:
- GF_SECURITY_ADMIN_PASSWORD=pass

114
docker/prometheus/prometheus.yml
View file

@ -3,9 +3,115 @@ global:
external_labels:
monitor: 'my-monitor'
scrape_configs:
- job_name: 'prometheus'
target_groups:
- targets: ['localhost:9090']
#- job_name: 'prometheus'
# target_groups:
# - targets: ['localhost:9090']
- job_name: 'node-exporter'
target_groups:
- targets: ['node-exporter:9100']
- targets: ['node-exporter:9100']
- job_name: 'app_servers'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/app-servers
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'app_times'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/app-times
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'classifiers'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/classifiers
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'db_times'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/db-times
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'errors'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/errors
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'feed_counts'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/feed-counts
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'feeds'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/feeds
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'load_times'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/load-times
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'stories'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/stories
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'task_codes'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/task-codes
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'task_pipeline'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/task-pipeline
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'task_servers'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/task-servers
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'task_times'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/task-times
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'updates'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/updates
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'users'
target_groups:
- targets: ['haproxy']
metrics_path: /monitor/users
scheme: https
tls_config:
insecure_skip_verify: true

111
docker/prometheus/prometheus.yml.j2
View file

@ -5,7 +5,114 @@ global:
scrape_configs:
- job_name: 'prometheus'
target_groups:
- targets: ['localhost:9090']
- targets: ['localhost:9090']
- job_name: 'node-exporter'
target_groups:
- targets: ['node-exporter:9100']
- targets: ['node-exporter:9100']
- job_name: 'app_servers'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/app-servers
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'app_times'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/app-times
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'classifiers'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/classifiers
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'db_times'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/db-times
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'errors'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/errors
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'feed_counts'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/feed-counts
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'feeds'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/feeds
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'load_times'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/load-times
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'stories'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/stories
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'task_codes'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/task-codes
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'task_pipeline'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/task-pipeline
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'task_servers'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/task-servers
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'task_times'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/task-times
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'updates'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/updates
scheme: https
tls_config:
insecure_skip_verify: true
- job_name: 'users'
target_groups:
- targets: ['{{ ansible_ssh_host }}']
metrics_path: /monitor/users
scheme: https
tls_config:
insecure_skip_verify: true

3
templates/monitor/prometheus_data.html
View file

@ -1,3 +1,4 @@
{% for k, v in data.items %}
{{ k }} {{ v }}<br>
# TYPE {{chart_name}}_{{k}} {{chart_type}}
{{chart_name}}_{{k}} {{ v }}
{% endfor %}