NewsBlur-viq/apps/statistics/models.py

import datetime
import mongoengine as mongo
from django.db.models import Avg, Count
from apps.rss_feeds.models import MFeedFetchHistory, FeedLoadtime
from apps.profile.models import Profile
from utils import json_functions as json
class MStatistics(mongo.Document):
    key   = mongo.StringField(unique=True)
    value = mongo.StringField()
    
    meta = {
        'collection': 'statistics',
        'allow_inheritance': False,
        'indexes': ['key'],
    }
    
    def __unicode__(self):
        return "%s: %s" % (self.key, self.value)
    
    @classmethod
    def all(cls):
        values = dict([(stat.key, stat.value) for stat in cls.objects.all()])
        for key, value in values.items():
            if key in ('avg_time_taken', 'sites_loaded'):
                values[key] = json.decode(value)
            elif key in ('feeds_fetched', 'premium_users', 'standard_users', 'latest_sites_loaded',
                         'max_sites_loaded'):
                values[key] = int(value)
            elif key in ('latest_avg_time_taken', 'max_avg_time_taken'):
                values[key] = float(value)
                
        return values
        
    @classmethod
    def collect_statistics(cls):
        last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
        feeds_fetched = MFeedFetchHistory.objects(fetch_date__gte=last_day).count()
        cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched)
        
        premium_users = Profile.objects.filter(last_seen_on__gte=last_day, is_premium=True).count()
        cls.objects(key='premium_users').update_one(upsert=True, key='premium_users', value=premium_users)
        
        standard_users = Profile.objects.filter(last_seen_on__gte=last_day, is_premium=False).count()
        cls.objects(key='standard_users').update_one(upsert=True, key='standard_users', value=standard_users)

        now = datetime.datetime.now()
        sites_loaded = []
        avg_time_taken = []
        for hour in range(24):
            start_hours_ago = now - datetime.timedelta(hours=hour)
            end_hours_ago = now - datetime.timedelta(hours=hour+1)
            aggregates = dict(count=Count('loadtime'), avg=Avg('loadtime'))
            load_times = FeedLoadtime.objects.filter(
                date_accessed__lte=start_hours_ago, 
                date_accessed__gte=end_hours_ago
            ).aggregate(**aggregates)
            sites_loaded.append(load_times['count'] or 0)
            avg_time_taken.append(load_times['avg'] or 0)
        sites_loaded.reverse()
        avg_time_taken.reverse()
        cls.objects(key='sites_loaded').update_one(upsert=True, key='sites_loaded', value=json.encode(sites_loaded))
        cls.objects(key='avg_time_taken').update_one(upsert=True, key='avg_time_taken', value=json.encode(avg_time_taken))
        cls.objects(key='latest_sites_loaded').update_one(upsert=True, key='latest_sites_loaded', value=sites_loaded[-1])
        cls.objects(key='latest_avg_time_taken').update_one(upsert=True, key='latest_avg_time_taken', value=avg_time_taken[-1])
        print sites_loaded, avg_time_taken
        print max(sites_loaded), max(avg_time_taken)
        cls.objects(key='max_sites_loaded').update_one(upsert=True, key='max_sites_loaded', value=max(sites_loaded))
        cls.objects(key='max_avg_time_taken').update_one(upsert=True, key='max_avg_time_taken', value=max(1, max(avg_time_taken)))