Added support for customizing the bucket size, bug fixes
Sapan Bhatia [Tue, 12 Nov 2013 19:59:42 +0000 (14:59 -0500)]
aggmgr.py
mp.yaml
mp3600.yaml [new file with mode: 0644]
mp86400.yaml [new file with mode: 0644]
mpsubmit.py

index cbfc48b..5fab758 100644 (file)
--- a/aggmgr.py
+++ b/aggmgr.py
@@ -12,7 +12,7 @@ class AggManager:
                self.matches = {}
                for e in self.parsed:
                        e['compiled_regex'] = re.compile(r'%s'%e['regex'])
-                       self.props[e['name']] = {'ds':e['ds']}
+                       self.props[e['name']] = e
                        self.matches[e['name']]={}
        
        def match_on_file(self, f):
@@ -49,7 +49,22 @@ class AggManager:
                                for v in l:
                                        svar='s%d'%i0
                                        ds = self.props[n]['ds']
-                                       self.defs.append('DEF:%s=%s:%s:AVERAGE'%(svar,v,ds))
+                                       try:
+                                               print self.props[n]['step']     
+                                       except:
+                                               pass
+
+                                       try:
+                                               step = ':step=%r'%self.props[n]['step']
+                                       except:
+                                               step = ''
+                               
+                                       try:
+                                               cf = self.props[n]['cf']
+                                       except:
+                                               cf = 'AVERAGE'
+
+                                       self.defs.append('DEF:%s=%s:%s:%s%s'%(svar,v,ds,cf,step))
                                        if (psexp):
                                                psexp.extend([svar,'ADDNAN'])
                                        else:
diff --git a/mp.yaml b/mp.yaml
index aaf60d0..1ee1a42 100644 (file)
--- a/mp.yaml
+++ b/mp.yaml
@@ -21,3 +21,53 @@ entries:
   regex: node\d+\.princeton\.vicci\.org/interface/if_octets-public0.rrd
   ds: tx
   fn: sum
+
+- name: cdn_sent_5m
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_sent.rrd
+  ds: value
+  fn: sum
+
+- name: cdn_hit_5m
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_hit.rrd
+  ds: value
+  fn: sum
+
+- name: cdn_hit_pct_5m
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_hit_pct.rrd
+  ds: value
+  fn: sum
+
+- name: cdn_req_5m
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_req.rrd
+  ds: value
+  fn: sum
+
+- name: cdn_nginx_status
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-nginx_status.rrd
+  ds: value
+  fn: sum
+
+- name: cdn_prox_status
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-prox_status.rrd
+  ds: value
+  fn: sum
+
+- name: cdn_resolve_status
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-resolve_status.rrd
+  ds: value
+  fn: sum
+
+- name: cdn_nginx_response_time
+  regex: node\d+\.princeton\.vicci\.org/hpc/response_time-nginx_response.rrd
+  ds: value
+  fn: sum
+
+- name: cdn_prox_response_time
+  regex: node\d+\.princeton\.vicci\.org/hpc/response_time-prox_response.rrd
+  ds: value
+  fn: sum
+
+- name: cdn_resolve_response_time
+  regex: node\d+\.princeton\.vicci\.org/hpc/response_time-resolve_response.rrd
+  ds: value
+  fn: sum
diff --git a/mp3600.yaml b/mp3600.yaml
new file mode 100644 (file)
index 0000000..8544a5d
--- /dev/null
@@ -0,0 +1,27 @@
+name: princeton_aggregator
+output: xml
+
+entries:
+- name: cdn_sent_1h
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_sent.rrd
+  step: 3600
+  ds: value
+  fn: sum
+
+- name: cdn_hit_1h
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_hit.rrd
+  step: 3600
+  ds: value
+  fn: sum
+
+- name: cdn_hit_pct_1h
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_hit_pct.rrd
+  step: 3600
+  ds: value
+  fn: sum
+
+- name: cdn_req_1h
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_req.rrd
+  step: 3600
+  ds: value
+  fn: sum
diff --git a/mp86400.yaml b/mp86400.yaml
new file mode 100644 (file)
index 0000000..ed242c9
--- /dev/null
@@ -0,0 +1,27 @@
+name: princeton_aggregator
+output: xml
+
+entries:
+- name: cdn_sent_24h
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_sent.rrd
+  step: 86400
+  ds: value
+  fn: sum
+
+- name: cdn_hit_24h
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_hit.rrd
+  step: 86400
+  ds: value
+  fn: sum
+
+- name: cdn_hit_pct_24h
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_hit_pct.rrd
+  step: 86400
+  ds: value
+  fn: sum
+
+- name: cdn_req_24h
+  regex: node\d+\.princeton\.vicci\.org/hpc/gauge-cdn_5m_req.rrd
+  step: 86400
+  ds: value
+  fn: sum
index 2e716a2..1da4ddd 100755 (executable)
@@ -9,19 +9,19 @@ import pdb
 import glob
 
 def track(event, properties=None):
-    if properties == None:
-        properties = {}
-    token = "blanked"XXX
-    if "token" not in properties:
-        properties["token"] = token
+       if properties == None:
+               properties = {}
+       token = "2504af1a1d3eee3f1744a764807d89b0"
+       if "token" not in properties:
+               properties["token"] = token
 
-    params = {"event": event, "properties": properties}
-    data = base64.b64encode(json.dumps(params))
-    request = "http://api.mixpanel.com/track/"
-    result = requests.get(request,params={'data':data})
-    return result
+       params = {"event": event, "properties": properties}
+       data = base64.b64encode(json.dumps(params))
+       request = "http://api.mixpanel.com/track/"
+       result = requests.get(request,params={'data':data})
+       return result
 
-cdata = os.popen('./rrdaggregate.py mp.yaml').read()
+cdata = os.popen('/etc/rrdaggregate/rrdaggregate.py /etc/rrdaggregate/mp.yaml').read()
 d = xmltodict.parse(cdata)
 legend = d['xport']['meta']['legend']['entry']
 data = d['xport']['data']['row'][-3]['v'] 
@@ -29,29 +29,32 @@ time = d['xport']['data']['row'][-3]['t']
 pdata = []
 
 for e in data:
-    pdata.append(float(e))
+       pdata.append(float(e))
   
 if (len(legend)!=len(data)):
-    'Mismatch between legend and data'
+       'Mismatch between legend and data'
 else:
-    prop_list = zip(legend,pdata)
-    props = dict(prop_list)
-    slices = glob.glob('/vservers/*')
-    slices = map(lambda s:s[10:],slices)
-    props['hostname']=os.popen('hostname').read().rstrip()
-    try:
-        props['memory']/=(1024*1024)
-    except:
-        pass 
-    props['ip']=os.popen('hostname -i').read().rstrip()
-    props['slices']=slices
-    
-    try:
-       last_sent = open('/tmp/lst_mp_hb').read()
-    except:
-       last_sent = '0'
+       prop_list = zip(legend,pdata)
+       props = dict(prop_list)
+       slices = glob.glob('/vservers/*')
+       slices = map(lambda s:s[10:],slices)
+       props['hostname']=os.popen('hostname').read().rstrip()
+       props['kernel']=os.popen('uname -r').read().rstrip()
+       try:
+               props['cpu']=100 - int(100*props['cpu']/12)/100.0;
+               props['memory']/=(1024*1024)
+       except:
+               pass 
+       props['ip']=os.popen('hostname -i').read().rstrip()
+       props['slices']=slices
+       
+       try:
+               last_sent = open('/tmp/lst_mp_hb').read()
+       except:
+               last_sent = '0'
 
-    if (last_sent!=time):
-        print 'Sending %r'%props
-        open('/tmp/lst_mp_hb','w').write('%s'%time)
-    
+       if (last_sent!=time):
+               print 'Sending'
+               track('heartbeat',props)
+               open('/tmp/lst_mp_hb','w').write('%s'%time)
+