Cleaning analytics as a beat task. Fixing server status codes in munin, and removing app feed fetches from analytics.

apps/reader/tasks.py

@@ -50,4 +50,19 @@ class CollectFeedback(Task):
     def run(self, **kwargs):
         logging.debug(" ---> Collecting feedback...")
         MFeedback.collect_feedback()
-        
+
+class CleanAnalytics(Task):
+    name = 'clean-analytics'
+
+    def run(self, **kwargs):
+        logging.debug(" ---> Cleaning analytics... %s page loads and %s feed fetches" % (
+            settings.MONGOANALYTICSDB.nbanalytics.page_loads.count(),
+            settings.MONGOANALYTICSDB.nbanalytics.feed_fetches.count(),
+        ))
+        day_ago = datetime.datetime.utcnow() - datetime.timedelta(days=1)
+        settings.MONGOANALYTICSDB.nbanalytics.feed_fetches.remove({
+            "date": {"$lt": day_ago},
+        })
+        settings.MONGOANALYTICSDB.nbanalytics.page_loads.remove({
+            "date": {"$lt": day_ago},
+        })

apps/statistics/models.py

@@ -339,6 +339,9 @@ class MAnalyticsFetcher(mongo.Document):
     @classmethod
     def add(cls, feed_id, feed_fetch, feed_process, 
             page, icon, total, feed_code):
+        server_name = settings.SERVER_NAME
+        if 'app' in server_name: return
+        
         if icon and page:
             icon -= page
         if page and feed_process:
@@ -347,7 +350,6 @@ class MAnalyticsFetcher(mongo.Document):
             page -= feed_fetch
         if feed_process and feed_fetch:
             feed_process -= feed_fetch
-        server_name = settings.SERVER_NAME
         
         cls.objects.create(feed_id=feed_id, feed_fetch=feed_fetch,
                            feed_process=feed_process, 

settings.py

@@ -352,6 +352,11 @@ CELERYBEAT_SCHEDULE = {
         'schedule': datetime.timedelta(hours=1),
         'options': {'queue': 'beat_tasks'},
     },
+    'clean-analytics': {
+        'task': 'clean-analytics',
+        'schedule': datetime.timedelta(hours=12),
+        'options': {'queue': 'beat_tasks'},
+    },
 }
 
 # =========

utils/munin/newsblur_tasks_codes.py

@@ -10,12 +10,12 @@ class NBMuninGraph(MuninGraph):
             'graph_title' : 'NewsBlur Task Codes',
             'graph_vlabel' : 'Status codes on feed fetch',
         }
-        servers = dict((("%s.label" % s['_id'], s['_id']) for s in self.stats))
+        servers = dict((("_%s.label" % s['_id'], s['_id']) for s in self.stats))
         graph.update(servers)
         return graph
 
     def calculate_metrics(self):
-        servers = dict((("%s" % s['_id'], s['feeds']) for s in self.stats))
+        servers = dict((("_%s" % s['_id'], s['feeds']) for s in self.stats))
         return servers
     
     @property