Adding analytics cleaning task.

apps/statistics/models.py

@@ -7,6 +7,7 @@ from apps.social.models import MSharedStory
 from apps.profile.models import Profile
 from utils import json_functions as json
 from utils import db_functions
+from utils import log as logging
 
 class MStatistics(mongo.Document):
     key   = mongo.StringField(unique=True)
@@ -318,7 +319,7 @@ class MAnalyticsPageLoad(mongo.Document):
         try:
             delete_old_history()
         except TimeoutError:
-            print "Timed out on deleting old history. Shit."
+            logging.debug("~SK~SB~BR~FWTimed out on deleting old page load history. Shit.")
         
 
 class MAnalyticsFetcher(mongo.Document):
@@ -383,5 +384,5 @@ class MAnalyticsFetcher(mongo.Document):
         try:
             delete_old_history()
         except TimeoutError:
-            print "Timed out on deleting old history. Shit."
+            logging.debug("~SK~SB~BR~FWTimed out on deleting old fetch history. Shit.")
         

apps/statistics/tasks.py

@@ -1,6 +1,7 @@
 from celery.task import Task
 from apps.statistics.models import MStatistics
 from apps.statistics.models import MFeedback
+from apps.statistics.models import MAnalyticsPageLoad, MAnalyticsFetcher
 from utils import log as logging
 
 
@@ -19,3 +20,12 @@ class CollectFeedback(Task):
     def run(self, **kwargs):
         logging.debug(" ---> ~FMCollecting feedback...")
         MFeedback.collect_feedback()
+
+class CleanAnalytics(Task):
+    name = 'clean-analytics'
+
+    def run(self, **kwargs):
+        logging.debug(" ---> ~FMCleaning analytics...")
+        MAnalyticsFetcher.clean()
+        MAnalyticsPageLoad.clean()
+        logging.debug(" ---> ~FMDone cleaning analytics...")

config/spawn_fcgi_munin_graph.conf

@@ -0,0 +1,119 @@
+#! /bin/sh
+
+### BEGIN INIT INFO
+# Provides:          spawn-fcgi-munin-graph
+# Required-Start:    $all
+# Required-Stop:     $all
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Description:       starts FastCGI for Munin-Graph
+### END INIT INFO
+# --------------------------------------------------------------
+# Munin-CGI-Graph Spawn-FCGI Startscript by Julien Schmidt
+# eMail: munin-trac at julienschmidt.com
+# www:   http://www.julienschmidt.com
+# --------------------------------------------------------------
+# Install: 
+#   1. Copy this file to /etc/init.d
+#   2. Edit the variables below
+#   3. run "update-rc.d spawn-fcgi-munin-graph defaults"
+# --------------------------------------------------------------
+# Last Update: 13. November 2012
+#
+# Please change the following variables:
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+NAME=spawn-fcgi-munin-graph
+PID_FILE=/var/run/munin/fcgi-graph.pid
+SOCK_FILE=/var/run/munin/fcgi-graph.sock
+DAEMON=/usr/bin/spawn-fcgi
+DAEMON_OPTS="-s $SOCK_FILE -U www-data -u www-data -g www-data /usr/lib/cgi-bin/munin-cgi-graph -P $PID_FILE"
+
+# --------------------------------------------------------------
+# No edits necessary beyond this line
+# --------------------------------------------------------------
+
+if [ ! -x $DAEMON ]; then
+	echo "File not found or is not executable: $DAEMON!"
+	exit 0
+fi
+
+status() {
+	if [ ! -r $PID_FILE ]; then
+		return 1
+	fi
+	
+	FCGI_PID=`cat $PID_FILE`
+	if [ -z "${FCGI_PID}" ]; then
+		return 1
+	fi
+	
+	FCGI_RUNNING=`ps -p ${FCGI_PID} | grep ${FCGI_PID}`
+	if [ -z "${FCGI_RUNNING}" ]; then
+		return 1
+	fi
+	
+	return 0
+}
+	
+start() {
+	if status; then
+		echo "FCGI is already running!"
+		exit 1
+	else
+		$DAEMON $DAEMON_OPTS
+	fi
+}
+
+stop () {	
+	if ! status; then
+		echo "No PID-file at $PID_FILE found or PID not valid. Maybe not running"
+		exit 1
+	fi
+	
+	# Kill process
+	kill -9 `cat $PID_FILE`
+	
+	# Remove PID-file
+	rm -f $PID_FILE
+	
+	# Remove Sock-File
+	rm -f $SOCK_FILE
+}
+
+case "$1" in
+	start)
+		echo "Starting $NAME: "
+		start
+		echo "... DONE"
+	;;
+
+	stop)
+		echo "Stopping $NAME: "
+		stop
+		echo "... DONE"
+	;;
+
+	force-reload|restart)
+		echo "Stopping $NAME: "
+		stop
+		echo "Starting $NAME: "
+		start
+		echo "... DONE"
+	;;
+	
+	status)
+		if status; then
+			echo "FCGI is RUNNING"
+		else
+			echo "FCGI is NOT RUNNING"
+		fi
+	;;
+	
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 1
+		;;
+esac
+
+exit 0

fabfile.py

@@ -301,6 +301,7 @@ def setup_common():
     setup_logrotate()
     setup_nginx()
     configure_nginx()
+    setup_munin()
 
 def setup_app():
     setup_common()
@@ -351,13 +352,12 @@ def setup_task():
 def setup_installs():
     sudo('apt-get -y update')
     sudo('apt-get -y upgrade')
-    sudo('apt-get -y install build-essential gcc scons libreadline-dev sysstat iotop git zsh python-dev locate python-software-properties libpcre3-dev libncurses5-dev libdbd-pg-perl libssl-dev make pgbouncer python-psycopg2 libmemcache0 python-memcache libyaml-0-2 python-yaml python-numpy python-scipy python-imaging munin munin-node munin-plugins-extra curl monit')
+    sudo('apt-get -y install build-essential gcc scons libreadline-dev sysstat iotop git zsh python-dev locate python-software-properties libpcre3-dev libncurses5-dev libdbd-pg-perl libssl-dev make pgbouncer python-psycopg2 libmemcache0 python-memcache libyaml-0-2 python-yaml python-numpy python-scipy python-imaging curl monit')
     # sudo('add-apt-repository ppa:pitti/postgresql')
     sudo('apt-get -y update')
     sudo('apt-get -y install postgresql-client')
     sudo('mkdir -p /var/run/postgresql')
     sudo('chown postgres.postgres /var/run/postgresql')
-    put('config/munin.conf', '/etc/munin/munin.conf', use_sudo=True)
     with settings(warn_only=True):
         run('git clone git://github.com/robbyrussell/oh-my-zsh.git ~/.oh-my-zsh')
     run('curl -O http://peak.telecommunity.com/dist/ez_setup.py')
@@ -720,6 +720,16 @@ def setup_redis():
     sudo('/etc/init.d/redis stop')
     sudo('/etc/init.d/redis start')
 
+def setup_munin():
+    sudo('apt-get update')
+    sudo('apt-get install -y munin munin-node munin-plugins-extra spawn-fcgi')
+    put('config/munin.conf', '/etc/munin/munin.conf', use_sudo=True)
+    put('config/spawn_fcgi_munin_graph.conf', '/etc/init.d/spawn_fcgi_munin_graph', use_sudo=True)
+    sudo('chmod u+x /etc/init.d/spawn_fcgi_munin_graph')
+    sudo('/etc/init.d/spawn_fcgi_munin_graph start')
+    sudo('update-rc.d spawn_fcgi_munin_graph defaults')
+
+    
 def setup_db_munin():
     sudo('cp -frs %s/config/munin/mongo* /etc/munin/plugins/' % env.NEWSBLUR_PATH)
     sudo('cp -frs %s/config/munin/pg_* /etc/munin/plugins/' % env.NEWSBLUR_PATH)

settings.py

@@ -367,6 +367,11 @@ CELERYBEAT_SCHEDULE = {
         'schedule': datetime.timedelta(hours=24),
         'options': {'queue': 'beat_tasks'},
     },
+    'clean-analytics': {
+        'task': 'clean-analytics',
+        'schedule': datetime.timedelta(hours=24),
+        'options': {'queue': 'beat_tasks'},
+    },
 }
 
 # =========