Script for setting up Mixpanel with rrdtool
Sapan Bhatia [Fri, 8 Nov 2013 01:33:32 +0000 (20:33 -0500)]
rrdagg.yaml [new file with mode: 0644]
rrdagg/aggmgr.py [new file with mode: 0644]
rrdagg/mpsubmit.py [new file with mode: 0755]
rrdagg/mpsubmit.py.orig [new file with mode: 0755]
rrdagg/rrdaggregate.py [new file with mode: 0755]

diff --git a/rrdagg.yaml b/rrdagg.yaml
new file mode 100644 (file)
index 0000000..769ec54
--- /dev/null
@@ -0,0 +1,38 @@
+---
+- hosts: princeton-vicci
+  user: root
+
+  tasks:
+    - name: install pip
+      yum: name={{ item }} state=latest
+      with_items:
+        - python-pip
+        - python-requests
+        - PyYAML
+
+    - name: install xmltodict
+      pip: name=xmltodict
+
+    - name: make script dir
+      shell: mkdir -p /etc/rrdaggregate
+
+    - name: copy rrdaggregate.py
+      copy: src=rrdagg/rrdaggregate.py dest=/etc/rrdaggregate/rrdaggregate.py
+      
+    - name: copy aggmgr.py
+      copy: src=rrdagg/aggmgr.py dest=/etc/rrdaggregate/aggmgr.py
+
+    - name: copy mpsubmit.py
+      copy: src=rrdagg/mpsubmit.py dest=/etc/rrdaggregate/mpsubmit.py
+      tags:
+        - mp
+
+    - name: copy mp.yaml
+      copy: src=rrdagg/mp.yaml dest=/etc/rrdaggregate/mp.yaml
+      tags:
+        - mp
+
+    - name: cron job
+      cron: name="run rrdagg" minute=* job=/etc/rrdaggregate/mpsubmit.py
+      tags:
+        - mp
diff --git a/rrdagg/aggmgr.py b/rrdagg/aggmgr.py
new file mode 100644 (file)
index 0000000..cbfc48b
--- /dev/null
@@ -0,0 +1,61 @@
+import yaml
+import os
+import re
+
+FIND_PATH='find'
+
+class AggManager:
+       def __init__(self, program):
+               self.parsed = yaml.load(open(program))['entries']
+               self.props = {}
+
+               self.matches = {}
+               for e in self.parsed:
+                       e['compiled_regex'] = re.compile(r'%s'%e['regex'])
+                       self.props[e['name']] = {'ds':e['ds']}
+                       self.matches[e['name']]={}
+       
+       def match_on_file(self, f):
+               for e in self.parsed:
+                       r = e['compiled_regex']
+                       m = re.match(r, f)
+                       if m:
+                               key_path = [e['name']] + list(m.groups())
+                               key = '###'.join(key_path)
+                               try:
+                                       self.matches[e['name']][key].append(f)
+                               except KeyError:
+                                       self.matches[e['name']][key]=[f]
+
+       def match_on_directory(self, dirname):
+               cmdline = '%s %s -printf "%%P\n"'%(FIND_PATH, dirname)
+               files = [line.rstrip() for line in os.popen(cmdline)]
+
+               for f in files:
+                       self.match_on_file(f)
+
+       def gen_defs(self):
+               i0 = 0
+               i1 = 0
+               self.defs = []
+               self.cdefs = []
+               self.xports = []
+               for e in self.parsed:
+                       n = e['name']
+                       psexp = []
+                       
+                       for k,l in self.matches[n].items():
+                               xvar='x%d'%i1
+                               for v in l:
+                                       svar='s%d'%i0
+                                       ds = self.props[n]['ds']
+                                       self.defs.append('DEF:%s=%s:%s:AVERAGE'%(svar,v,ds))
+                                       if (psexp):
+                                               psexp.extend([svar,'ADDNAN'])
+                                       else:
+                                               psexp=[svar]
+                                       i0+=1
+
+                               self.cdefs.append('CDEF:%s=%s'%(xvar,','.join(psexp)))
+                               i1+=1
+                               self.xports.append('XPORT:%s:"%s"'%(xvar,k))
diff --git a/rrdagg/mpsubmit.py b/rrdagg/mpsubmit.py
new file mode 100755 (executable)
index 0000000..1da4ddd
--- /dev/null
@@ -0,0 +1,60 @@
+#!/usr/bin/python
+
+import base64
+import requests
+import json
+import os
+import xmltodict
+import pdb
+import glob
+
+def track(event, properties=None):
+       if properties == None:
+               properties = {}
+       token = "2504af1a1d3eee3f1744a764807d89b0"
+       if "token" not in properties:
+               properties["token"] = token
+
+       params = {"event": event, "properties": properties}
+       data = base64.b64encode(json.dumps(params))
+       request = "http://api.mixpanel.com/track/"
+       result = requests.get(request,params={'data':data})
+       return result
+
+cdata = os.popen('/etc/rrdaggregate/rrdaggregate.py /etc/rrdaggregate/mp.yaml').read()
+d = xmltodict.parse(cdata)
+legend = d['xport']['meta']['legend']['entry']
+data = d['xport']['data']['row'][-3]['v'] 
+time = d['xport']['data']['row'][-3]['t'] 
+pdata = []
+
+for e in data:
+       pdata.append(float(e))
+  
+if (len(legend)!=len(data)):
+       'Mismatch between legend and data'
+else:
+       prop_list = zip(legend,pdata)
+       props = dict(prop_list)
+       slices = glob.glob('/vservers/*')
+       slices = map(lambda s:s[10:],slices)
+       props['hostname']=os.popen('hostname').read().rstrip()
+       props['kernel']=os.popen('uname -r').read().rstrip()
+       try:
+               props['cpu']=100 - int(100*props['cpu']/12)/100.0;
+               props['memory']/=(1024*1024)
+       except:
+               pass 
+       props['ip']=os.popen('hostname -i').read().rstrip()
+       props['slices']=slices
+       
+       try:
+               last_sent = open('/tmp/lst_mp_hb').read()
+       except:
+               last_sent = '0'
+
+       if (last_sent!=time):
+               print 'Sending'
+               track('heartbeat',props)
+               open('/tmp/lst_mp_hb','w').write('%s'%time)
+       
diff --git a/rrdagg/mpsubmit.py.orig b/rrdagg/mpsubmit.py.orig
new file mode 100755 (executable)
index 0000000..3cee8ac
--- /dev/null
@@ -0,0 +1,57 @@
+#!/usr/bin/python
+
+import base64
+import requests
+import json
+import os
+import xmltodict
+import pdb
+import glob
+
+def track(event, properties=None):
+    if properties == None:
+        properties = {}
+    token = "2504af1a1d3eee3f1744a764807d89b0"
+    if "token" not in properties:
+        properties["token"] = token
+
+    params = {"event": event, "properties": properties}
+    data = base64.b64encode(json.dumps(params))
+    request = "http://api.mixpanel.com/track/"
+    result = requests.get(request,params={'data':data})
+    return result
+
+cdata = os.popen('./rrdaggregate.py mp.yaml').read()
+d = xmltodict.parse(cdata)
+legend = d['xport']['meta']['legend']['entry']
+data = d['xport']['data']['row'][-3]['v'] 
+time = d['xport']['data']['row'][-3]['t'] 
+pdata = []
+
+for e in data:
+    pdata.append(float(e))
+  
+if (len(legend)!=len(data)):
+    'Mismatch between legend and data'
+else:
+    prop_list = zip(legend,pdata)
+    props = dict(prop_list)
+    slices = glob.glob('/vservers/*')
+    slices = map(lambda s:s[10:],slices)
+    props['hostname']=os.popen('hostname').read().rstrip()
+    try:
+        props['memory']/=(1024*1024)
+    except:
+        pass 
+    props['ip']=os.popen('hostname -i').read().rstrip()
+    props['slices']=slices
+    
+    try:
+       last_sent = open('/tmp/lst_mp_hb').read()
+    except:
+       last_sent = '0'
+
+    if (last_sent!=time):
+        print 'Sending %r'%props
+        open('/tmp/lst_mp_hb','w').write('%s'%time)
+    
diff --git a/rrdagg/rrdaggregate.py b/rrdagg/rrdaggregate.py
new file mode 100755 (executable)
index 0000000..e9a3246
--- /dev/null
@@ -0,0 +1,38 @@
+#!/usr/bin/python
+
+import yaml
+import argparse
+import os
+from aggmgr import AggManager
+
+expressions="""
+(node\d+.princeton.vicci.org)/cpu-\d/.*
+"""
+
+def main():
+       parser = argparse.ArgumentParser()
+       parser.add_argument('-datadir',help='Directory location of your rrd files',default='/usr/var/lib/collectd/rrd')
+       parser.add_argument('-rrdopts',help='Option string passed verbatim to rrdtool',default='--start now-1h --end now --step 300')
+       parser.add_argument('-pretend',help='Debug only',default=False,action='store_true')
+       parser.add_argument('program',help='Program in YAML that specifies the required aggregation.')
+       values = parser.parse_args()
+
+       aggp = AggManager(values.program)
+       aggp.match_on_directory(values.datadir)
+       aggp.gen_defs()
+
+       options = {'verbatim':values.rrdopts, 'defs':' '.join(aggp.defs), 'cdefs':' '.join(aggp.cdefs), 'xports':' '.join(aggp.xports)}
+       os.chdir(values.datadir)
+       cmdline = """
+       rrdtool xport %(verbatim)s %(defs)s %(cdefs)s %(xports)s
+       """%options
+
+       if (not values.pretend):
+               os.system(cmdline)
+       else:
+               print cmdline
+       
+
+
+if __name__=='__main__':
+       main ()