Merge branch 'master' into prometheus_retention

* master:
  Turning refresh stale feeds back on.
  Terraform Digital Ocean v2.0
  Adjusting sizes
  Better topline metrics
  Cleaning up redis dashboard.
  Flask metrics may not have the port, use default.

ansible/playbooks/setup_redis.yml

@@ -16,5 +16,5 @@
     - {role: 'consul-client', tags: 'consul'}
     - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
     - {role: 'redis', tags: 'redis'}
-    - {role: 'flask_metrics', tags: ['flask-metrics', 'metrics']}
+    - {role: 'flask_metrics', tags: ['flask-metrics', 'metrics', 'flask_metrics']}
     - {role: 'monitor', tags: 'monitor'}

apps/profile/middleware.py

@@ -21,10 +21,9 @@ class LastSeenMiddleware(object):
              request.path.startswith('/reader/load_feeds') or
              request.path.startswith('/reader/feeds'))
             and hasattr(request, 'user')
-            and request.user.is_authenticated): 
+            and request.user.is_authenticated):
             hour_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=60)
             ip = request.META.get('HTTP_X_FORWARDED_FOR', None) or request.META['REMOTE_ADDR']
-            # SUBSCRIBER_EXPIRE = datetime.datetime.utcnow() - datetime.timedelta(days=settings.SUBSCRIBER_EXPIRE)
             if request.user.profile.last_seen_on < hour_ago:
                 logging.user(request, "~FG~BBRepeat visitor: ~SB%s (%s)" % (
                     request.user.profile.last_seen_on, ip))
@@ -101,7 +100,8 @@ class DBProfilerMiddleware:
             self._save_times(self.sql_times_elapsed, 'app_')
     
     def _save_times(self, db_times, prefix=""):
-        if not db_times: return
+        if not db_times: 
+            return
         
         r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL)
         pipe = r.pipeline()

apps/profile/tasks.py

@@ -54,7 +54,7 @@ def CleanupUser(user_id):
     MActivity.trim(user_id)
     UserSubscriptionFolders.add_missing_feeds_for_user(user_id)
     UserSubscriptionFolders.compact_for_user(user_id)
-    # UserSubscription.refresh_stale_feeds(user_id)
+    UserSubscription.refresh_stale_feeds(user_id)
     
     try:
         ss = MSocialServices.objects.get(user_id=user_id)

docker/grafana/dashboards/redis_dashboard.json

@@ -8,16 +8,24 @@
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
         "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
         "type": "dashboard"
       }
     ]
   },
   "editable": true,
+  "fiscalYearStartMonth": 0,
   "gnetId": null,
   "graphTooltip": 0,
   "id": 8,
-  "iteration": 1638808327674,
+  "iteration": 1638880390648,
   "links": [],
+  "liveNow": false,
   "panels": [
     {
       "aliasColors": {},
@@ -25,10 +33,6 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -55,7 +59,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.5.7",
+      "pluginVersion": "8.2.6",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -98,7 +102,7 @@
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
@@ -122,10 +126,6 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -152,7 +152,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.5.7",
+      "pluginVersion": "8.2.6",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -195,7 +195,7 @@
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
@@ -219,10 +219,6 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -249,7 +245,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.5.7",
+      "pluginVersion": "8.2.6",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -287,14 +283,16 @@
       },
       "yaxes": [
         {
+          "$$hashKey": "object:176",
           "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
+          "$$hashKey": "object:177",
           "format": "short",
           "label": null,
           "logBase": 1,
@@ -314,10 +312,6 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -344,7 +338,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.5.7",
+      "pluginVersion": "8.2.6",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -382,14 +376,16 @@
       },
       "yaxes": [
         {
+          "$$hashKey": "object:406",
           "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
+          "$$hashKey": "object:407",
           "format": "short",
           "label": null,
           "logBase": 1,
@@ -409,10 +405,7 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
+      "description": "",
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -439,7 +432,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.5.7",
+      "pluginVersion": "8.2.6",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -450,7 +443,7 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "redis_commands{instance=\"$instance\"}",
+          "expr": "rate(redis_commands{instance=\"$instance\"}[1m])",
           "interval": "",
           "legendFormat": "Total Commands Processed",
           "queryType": "randomWalk",
@@ -461,7 +454,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Redis Commands",
+      "title": "Redis Commands per minute",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -477,14 +470,16 @@
       },
       "yaxes": [
         {
+          "$$hashKey": "object:562",
           "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
+          "$$hashKey": "object:563",
           "format": "short",
           "label": null,
           "logBase": 1,
@@ -505,10 +500,6 @@
       "dashes": false,
       "datasource": "Prometheus",
       "description": "",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -535,7 +526,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.5.7",
+      "pluginVersion": "8.2.6",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -595,7 +586,7 @@
       }
     }
   ],
-  "schemaVersion": 27,
+  "schemaVersion": 32,
   "style": "dark",
   "tags": [],
   "templating": {
@@ -603,10 +594,9 @@
       {
         "allValue": null,
         "current": {
-          "isNone": true,
           "selected": false,
-          "text": "None",
-          "value": ""
+          "text": "db-redis-pubsub",
+          "value": "db-redis-pubsub"
         },
         "datasource": "Prometheus",
         "definition": "label_values(redis_active_connections, instance)",
@@ -627,7 +617,6 @@
         "skipUrlSync": false,
         "sort": 0,
         "tagValuesQuery": "",
-        "tags": [],
         "tagsQuery": "",
         "type": "query",
         "useTags": false
@@ -642,5 +631,5 @@
   "timezone": "",
   "title": "Redis",
   "uid": "7IvPZkDnk",
-  "version": 1
+  "version": 3
 }

flask_metrics/flask_metrics_redis.py

@@ -35,7 +35,7 @@ class RedisMetric(object):
             if not settings.DOCKERBUILD and settings.SERVER_NAME != instance:
                 continue
             self.host = redis_config['host']
-            self.port = redis_config['port']
+            self.port = redis_config.get('port', settings.REDIS_PORT)
             stats = self.get_info()
             yield instance, stats
   

terraform/digitalocean.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     digitalocean = {
       source = "digitalocean/digitalocean"
-      version = "1.22.2"
+      version = "~> 2.0"
     }
   }