v0.1 Doesnt quite work
Sapan Bhatia [Fri, 25 Oct 2013 17:44:25 +0000 (13:44 -0400)]
README.md
aggmgr.py [new file with mode: 0644]
input.yaml [new file with mode: 0644]
rrdaggregate.py [new file with mode: 0755]

index cb3998e..a41b319 100644 (file)
--- a/README.md
+++ b/README.md
@@ -1,4 +1,8 @@
 rrdaggregate
 ============
 
-Generic aggregator of rrd data. Exports data in json, xml (but not rrd) format.
+rrdaggregate is a generic aggregator of rrd data. Exports data in json, xml (but not rrd) format.
+
+The input to rrdaggregate is an aggregation recipe, specified in yaml format:
+
+
diff --git a/aggmgr.py b/aggmgr.py
new file mode 100644 (file)
index 0000000..2ac74bb
--- /dev/null
+++ b/aggmgr.py
@@ -0,0 +1,62 @@
+import yaml
+import os
+import re
+
+FIND_PATH='/usr/bin/find'
+
+class AggManager:
+       def __init__(self, program):
+               self.parsed = yaml.load(open(program))['entries']
+
+               self.matches = {}
+               for e in self.parsed:
+                       e['compiled_regex'] = re.compile(r'%s'%e['regex'])
+                       self.matches[e['name']]={}
+       
+       def match_on_file(self, f):
+               for e in self.parsed:
+                       r = e['compiled_regex']
+                       m = re.match(r, f)
+                       if m:
+                               key = '###'.join(m.groups())
+                               try:
+                                       self.matches[e['name']][key].append(f)
+                               except KeyError:
+                                       self.matches[e['name']][key]=[f]
+
+       def match_on_directory(self, dirname):
+               cmdline = '%s %s -printf "%%P\n"'%(FIND_PATH, dirname)
+               files = [line.rstrip() for line in os.popen(cmdline)]
+
+               for f in files:
+                       self.match_on_file(f)
+
+       def gen_defs(self):
+               for e in self.parsed:
+                       n = e['name']
+                       i0 = 0
+                       i1 = 0
+                       self.defs = []
+                       self.cdefs = []
+                       self.xports = []
+                       psexp = []
+                       for k,l in self.matches[n].items():
+                               xvar='x%d'%i1
+                               for v in l:
+                                       svar='s%d'%i0
+                                       self.defs.append('DEF:%s=%s:value:AVERAGE'%(svar,v))
+                                       if (psexp):
+                                               psexp.extend([svar,'+'])
+                                       else:
+                                               psexp=[svar]
+                                       i0+=1
+
+                               self.cdefs.append('CDEF:%s=%s'%(xvar,','.join(psexp)))
+                               i1+=1
+                               self.xports.append('XPORT:%s:"%s"'%(xvar,xvar))
+                               
+                       
+
+
+
+
diff --git a/input.yaml b/input.yaml
new file mode 100644 (file)
index 0000000..ae1ad88
--- /dev/null
@@ -0,0 +1,16 @@
+name: princeton_aggregator
+output: json
+
+entries:
+- name: princeton_per_node_cpu
+  regex: (node\d+\.princeton\.vicci\.org)/cpu-\d+/cpu-idle.rrd
+  ds: value
+  fn: sum
+
+#- name: princeton_total_cpu
+#  regex: node\d+\.princeton\.vicci\.org/cpu-\d+
+#  fn: sum
+
+#- name: princeton_slice_cpu
+#  regex: (princeton_.*)/
+#  fn: sum
diff --git a/rrdaggregate.py b/rrdaggregate.py
new file mode 100755 (executable)
index 0000000..7e20bcf
--- /dev/null
@@ -0,0 +1,35 @@
+#!/usr/bin/python
+
+import yaml
+import argparse
+import os
+from aggmgr import AggManager
+
+expressions="""
+(node\d+.princeton.vicci.org)/cpu-\d/.*
+"""
+
+def main():
+       parser = argparse.ArgumentParser()
+       parser.add_argument('-datadir',help='Directory location of your rrd files',default='/usr/var/lib/collectd/rrd')
+       parser.add_argument('-rrdopts',help='Option string passed verbatim to rrdtool',default='--start now-1h --end now --json')
+       parser.add_argument('program',help='Program in YAML that specifies the required aggregation.')
+       values = parser.parse_args()
+
+       aggp = AggManager(values.program)
+       aggp.match_on_directory(values.datadir)
+       aggp.gen_defs()
+
+       options = {'verbatim':values.rrdopts, 'defs':' '.join(aggp.defs), 'cdefs':' '.join(aggp.cdefs), 'xports':' '.join(aggp.xports)}
+       #print "Defs = %s\nCDefs=%s\nXports=%s"%(aggp.defs,aggp.cdefs,aggp.xports)
+       os.chdir(values.datadir)
+       cmdline = """
+       rrdtool xport %(verbatim)s %(defs)s %(cdefs)s %(xports)s
+       """%options
+       os.system(cmdline)
+       #print cmdline
+       
+
+
+if __name__=='__main__':
+       main ()