Bug fixes + script to submit data to mixpanel
Sapan Bhatia [Wed, 30 Oct 2013 04:05:22 +0000 (00:05 -0400)]
aggmgr.py
mp.yaml [new file with mode: 0644]
mpsubmit.py [new file with mode: 0755]
rrdaggregate.py

index d556be7..cbfc48b 100644 (file)
--- a/aggmgr.py
+++ b/aggmgr.py
@@ -7,10 +7,12 @@ FIND_PATH='find'
 class AggManager:
        def __init__(self, program):
                self.parsed = yaml.load(open(program))['entries']
+               self.props = {}
 
                self.matches = {}
                for e in self.parsed:
                        e['compiled_regex'] = re.compile(r'%s'%e['regex'])
+                       self.props[e['name']] = {'ds':e['ds']}
                        self.matches[e['name']]={}
        
        def match_on_file(self, f):
@@ -18,7 +20,8 @@ class AggManager:
                        r = e['compiled_regex']
                        m = re.match(r, f)
                        if m:
-                               key = '###'.join(m.groups())
+                               key_path = [e['name']] + list(m.groups())
+                               key = '###'.join(key_path)
                                try:
                                        self.matches[e['name']][key].append(f)
                                except KeyError:
@@ -32,19 +35,21 @@ class AggManager:
                        self.match_on_file(f)
 
        def gen_defs(self):
+               i0 = 0
+               i1 = 0
+               self.defs = []
+               self.cdefs = []
+               self.xports = []
                for e in self.parsed:
                        n = e['name']
-                       i0 = 0
-                       i1 = 0
-                       self.defs = []
-                       self.cdefs = []
-                       self.xports = []
                        psexp = []
+                       
                        for k,l in self.matches[n].items():
                                xvar='x%d'%i1
                                for v in l:
                                        svar='s%d'%i0
-                                       self.defs.append('DEF:%s=%s:value:AVERAGE'%(svar,v))
+                                       ds = self.props[n]['ds']
+                                       self.defs.append('DEF:%s=%s:%s:AVERAGE'%(svar,v,ds))
                                        if (psexp):
                                                psexp.extend([svar,'ADDNAN'])
                                        else:
@@ -54,9 +59,3 @@ class AggManager:
                                self.cdefs.append('CDEF:%s=%s'%(xvar,','.join(psexp)))
                                i1+=1
                                self.xports.append('XPORT:%s:"%s"'%(xvar,k))
-                               
-                       
-
-
-
-
diff --git a/mp.yaml b/mp.yaml
new file mode 100644 (file)
index 0000000..aaf60d0
--- /dev/null
+++ b/mp.yaml
@@ -0,0 +1,23 @@
+name: princeton_aggregator
+output: xml
+
+entries:
+- name: cpu
+  regex: node\d+\.princeton\.vicci\.org/cpu-\d+/cpu-idle.rrd
+  ds: value
+  fn: sum
+
+- name: memory
+  regex: node\d+\.princeton\.vicci\.org/memory/memory-used.rrd
+  ds: value
+  fn: sum
+
+- name: disk
+  regex: node\d+\.princeton\.vicci\.org/disk-dm-\d+/disk_octets.rrd
+  ds: read
+  fn: sum
+
+- name: tx
+  regex: node\d+\.princeton\.vicci\.org/interface/if_octets-public0.rrd
+  ds: tx
+  fn: sum
diff --git a/mpsubmit.py b/mpsubmit.py
new file mode 100755 (executable)
index 0000000..2e716a2
--- /dev/null
@@ -0,0 +1,57 @@
+#!/usr/bin/python
+
+import base64
+import requests
+import json
+import os
+import xmltodict
+import pdb
+import glob
+
+def track(event, properties=None):
+    if properties == None:
+        properties = {}
+    token = "blanked"XXX
+    if "token" not in properties:
+        properties["token"] = token
+
+    params = {"event": event, "properties": properties}
+    data = base64.b64encode(json.dumps(params))
+    request = "http://api.mixpanel.com/track/"
+    result = requests.get(request,params={'data':data})
+    return result
+
+cdata = os.popen('./rrdaggregate.py mp.yaml').read()
+d = xmltodict.parse(cdata)
+legend = d['xport']['meta']['legend']['entry']
+data = d['xport']['data']['row'][-3]['v'] 
+time = d['xport']['data']['row'][-3]['t'] 
+pdata = []
+
+for e in data:
+    pdata.append(float(e))
+  
+if (len(legend)!=len(data)):
+    'Mismatch between legend and data'
+else:
+    prop_list = zip(legend,pdata)
+    props = dict(prop_list)
+    slices = glob.glob('/vservers/*')
+    slices = map(lambda s:s[10:],slices)
+    props['hostname']=os.popen('hostname').read().rstrip()
+    try:
+        props['memory']/=(1024*1024)
+    except:
+        pass 
+    props['ip']=os.popen('hostname -i').read().rstrip()
+    props['slices']=slices
+    
+    try:
+       last_sent = open('/tmp/lst_mp_hb').read()
+    except:
+       last_sent = '0'
+
+    if (last_sent!=time):
+        print 'Sending %r'%props
+        open('/tmp/lst_mp_hb','w').write('%s'%time)
+    
index 4e396b0..e9a3246 100755 (executable)
@@ -12,7 +12,7 @@ expressions="""
 def main():
        parser = argparse.ArgumentParser()
        parser.add_argument('-datadir',help='Directory location of your rrd files',default='/usr/var/lib/collectd/rrd')
-       parser.add_argument('-rrdopts',help='Option string passed verbatim to rrdtool',default='--start now-1h --end now --json')
+       parser.add_argument('-rrdopts',help='Option string passed verbatim to rrdtool',default='--start now-1h --end now --step 300')
        parser.add_argument('-pretend',help='Debug only',default=False,action='store_true')
        parser.add_argument('program',help='Program in YAML that specifies the required aggregation.')
        values = parser.parse_args()
@@ -22,7 +22,6 @@ def main():
        aggp.gen_defs()
 
        options = {'verbatim':values.rrdopts, 'defs':' '.join(aggp.defs), 'cdefs':' '.join(aggp.cdefs), 'xports':' '.join(aggp.xports)}
-       #print "Defs = %s\nCDefs=%s\nXports=%s"%(aggp.defs,aggp.cdefs,aggp.xports)
        os.chdir(values.datadir)
        cmdline = """
        rrdtool xport %(verbatim)s %(defs)s %(cdefs)s %(xports)s