import repository from arizona
[raven.git] / apps / errorlog / index.py
1 #!/usr/bin/env python
2 """
3 Author: Scott Baker
4 Date: 2010-8-3
5
6 Error logging server
7
8 This script will require:
9 mysql
10 mod_python
11 python-MySQLdb
12 python-json
13 """
14
15 import os
16 import sys
17 from errlogging import *
18
19 protocol = "1.2"
20
21 # maximum number of entries in the database
22 TRIM_LEVEL = 100000
23
24 config = {}
25 config["userpasswordfile"] = "/root/mysql-errorlog-pass.txt"
26 config["userpassword"] = None
27 config["rootpasswordfile"] = "/root/mysql-root-pass.txt"
28 config["rootpassword"] = None
29 config["host"] = "localhost"
30 config["user"] = "errorlog"
31 config["url"] = "/errorlog/"
32 config["path"] = "/usr/local/errorlog"
33 config["dbname"] = "errorlog"
34
35 from mod_python import apache, psp, util, Session
36 from cgi import escape
37 from urllib import unquote
38 from distutils.version import LooseVersion
39 from html import *
40 import json as json_module # avoid name conflict with json function
41 import MySQLdb
42 import ConfigParser
43 import traceback
44 import time
45 import string
46 import urllib
47 from db import *
48
49
50 # DB connection info
51 DB.host = config["host"]
52 DB.user = config["user"]
53 DB.name = None
54 DB.passwd = config["userpassword"]
55 if DB.passwd == None:
56     try:
57         file = config["userpasswordfile"]
58         fd = open(file, "r")
59         DB.passwd = fd.readline().rstrip()
60         fd.close()
61     except:
62         error(None, 'Unable to read mysql errorlog password from "%s"' % (file))
63         error(None, 'euid = %s, egid = %s' % (os.geteuid(), os.getegid()))
64         raise
65
66 def _trim(req, dbconn):
67     global config
68
69     if (dbconn == None):
70         db = config["dbname"]
71         dbconn = DB(name = db)
72
73     query = "SELECT id FROM `errors` ORDER BY id DESC LIMIT 1;"
74
75     dbconn.dict_cursor()
76     items = dbconn.execute(query).fetchall()
77
78     # lets hope we get more than one
79     if (len(items) <= 0):
80         return
81
82     item = items[0]
83
84     # 'id' had better be in there
85     if not ("id" in item):
86         return
87
88     highest_id = int(item["id"])
89
90     if (req != None):
91         req.content_type = "text/html"
92         req.write("Highest id is: " + str(highest_id) + "<br><br>")
93
94     if (highest_id > TRIM_LEVEL):
95         query = "DELETE FROM `errors` WHERE id<" + str(highest_id - TRIM_LEVEL) + ";"
96         if (req != None):
97             req.write("Executing: " + query + "<br><br>")
98         dbconn.execute(query)
99         if (req != None):
100             req.write("Done")
101
102 def trim(req):
103     """ simple wrapper around _trim, for debugging the output """
104     _trim(req, None)
105
106
107 def report(req, **args):
108     global config
109
110     db = config["dbname"]
111
112     # create the database if it does not exist
113     _create_db(req, db)
114
115     dbconn = None
116     try:
117         req.content_type = "text/html"
118
119         dbconn = DB(name = db)
120
121         _trim(None, dbconn)
122
123         dbconn.execute("""INSERT INTO `errors`
124                           (`program`, `version`, `hostname`, `trace`, `time`, `cmdline`, `username`, `uname`, `savefilename`, `exception`, `lineno`, `filename`)
125                           VALUES( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s );""",
126                             (str(args.get("program", "")),
127                              str(args.get("version", "")),
128                              str(args.get("hostname", "")),
129                              str(args.get("trace", "")),
130                              str(args.get("time", "")),
131                              str(args.get("cmdline", "")),
132                              str(args.get("username", "")),
133                              str(args.get("uname", "")),
134                              str(args.get("savefilename", "")),
135                              str(args.get("exception", "")),
136                              str(args.get("lineno", "")),
137                              str(args.get("filename", ""))))
138
139         req.write("OK")
140
141     finally:
142         if dbconn != None:
143             dbconn.close()
144
145 def reduce_items(items):
146     """ Given a list of items, eliminate duplicates that have the same
147         hostname, username, and trace.
148     """
149
150     results = []
151     dict = {}
152     for item in items:
153        key = item.get("hostname", "") + item.get("username", "") + item.get("trace", "")
154        if key in dict:
155            dict_item = dict[key]
156            dict_item["qty"] = dict_item["qty"] + 1
157        else:
158            item["qty"] = 1
159            dict[key] = item
160            results.append(item)
161
162     return results
163
164 def build_header(reduce=False, limit="100", search=None, exclude=None):
165     return FORM(config["url"], "GET",
166                   "search: " + INPUT("search", "text", 30, search) +
167                   " limit: " + INPUT("limit", "text", 10, limit) +
168                   " merge dups: " + SELECT("reduce", OPTION("yes", reduce, "yes") + OPTION("no", not reduce, "no")) +
169                   BR() +
170                   "exclude: " + INPUT("exclude", "text", 30, exclude) +
171                   BR() +
172                   BUTTON("submit", "submit"))
173
174 def _like(what, s):
175     return "(" + what + " like '%" + s + "%')"
176
177 def _notlike(what ,s):
178     return "(" + what + " not like '%" + s + "%')"
179
180 def _search_clause(keys, search, exclude):
181     conds = []
182     for str in search.split(" "):
183         str = str.strip()
184         if str:
185             for key in keys:
186                 conds.append( _like(key, str) )
187
188     exclude_conds = []
189     for str in exclude.split(" "):
190         str = str.strip()
191         if str:
192             for key in keys:
193                 exclude_conds.append( _notlike(key, str) )
194
195     if conds and exclude_conds:
196         where = "WHERE (" + " OR ".join(conds) + ") AND (" + " AND ".join(exclude_conds) + ")"
197     elif conds:
198         where = "WHERE " + " OR ".join(conds)
199     elif exclude_conds:
200         where = "WHERE " + " AND ".join(exclude_conds)
201     else:
202         where = ""
203
204     return where
205
206 def index(req, reduce="yes", limit="250", search=None, exclude=None):
207     global config
208
209     db = config["dbname"]
210
211     reduce = (reduce == "yes")
212
213     # create the database if it does not exist
214     _create_db(req, db)
215
216     dbconn = None
217     try:
218         req.content_type = "text/html"
219
220         req.write(build_header(reduce, limit, search, exclude) + "<br>")
221
222         dbconn = DB(name = db)
223
224         query = "SELECT * FROM `errors`"
225         if search or exclude:
226             query = query + " " + _search_clause(["hostname", "username", "exception", "program", "version", "filename", "lineno", "trace"], search, exclude)
227         query = query + " ORDER BY id DESC"
228         if limit:
229             query = query + " LIMIT " + str(limit)
230
231         req.write("query-executed: " + query + "<br>")
232
233         dbconn.dict_cursor()
234         items = dbconn.execute(query).fetchall()
235         #items = dbconn.execute("SELECT * FROM `errors` ORDER BY id DESC LIMIT " + str(limit) + ";").fetchall()
236
237         if reduce:
238            items = reduce_items(items)
239
240         req.write("<TABLE border=1>")
241
242         req.write(TR(TH("id") +
243                      TH("qty") +
244                      TH("hostname") +
245                      TH("username") +
246                      TH("exception") +
247                      TH("program") +
248                      TH("version") +
249                      TH("filename") +
250                      TH("lineno")
251                      ))
252
253         for item in items:
254              link = config['url'] + "detail?id=" + str(item.get("id"))
255              req.write(TR(TD(LINK(link,item.get("id", ""))) +
256                           TD(item.get("qty", "")) +
257                           TD(item.get("hostname", "")) +
258                           TD(item.get("username", "")) +
259                           TD(item.get("exception", "")) +
260                           TD(item.get("program", "")) +
261                           TD(item.get("version", "")) +
262                           TD(item.get("filename", "")) +
263                           TD(item.get("lineno", ""))
264                           ))
265
266         req.write("</TABLE>")
267
268     finally:
269         if dbconn != None:
270             dbconn.close()
271
272 def detail(req, id):
273     global config
274
275     db = config["dbname"]
276
277     # create the database if it does not exist
278     _create_db(req, db)
279
280     dbconn = None
281     try:
282         req.content_type = "text/html"
283
284         dbconn = DB(name = db)
285
286         dbconn.dict_cursor()
287         items = dbconn.execute("SELECT * FROM `errors` WHERE ID = %s;", id).fetchall()
288         item = items[0]
289
290         req.write("<TABLE>")
291         req.write(TR(TD("id:") + TD(item.get("id",""))))
292         req.write(TR(TD("exception:") + TD(item.get("exception",""))))
293         req.write(TR(TD("program:") + TD(item.get("program",""))))
294         req.write(TR(TD("version:") + TD(item.get("version",""))))
295         req.write(TR(TD("hostname:") + TD(item.get("hostname",""))))
296         req.write(TR(TD("username:") + TD(item.get("username",""))))
297         req.write(TR(TD("lineno:") + TD(item.get("lineno",""))))
298         req.write(TR(TD("filename:") + TD(item.get("filename",""))))
299         req.write(TR(TD("time:") + TD(item.get("time",""))))
300         req.write(TR(TD("cmdline:") + TD(item.get("cmdline",""))))
301         req.write("</TABLE>")
302
303         req.write("<PRE>")
304         req.write(item.get("trace"))
305         req.write("</PRE>")
306
307     finally:
308         if dbconn != None:
309             dbconn.close()
310
311
312 def _create_db(req, db):
313     global request
314
315     request = req
316     debug(req, "Creating database %s" % (db))
317     dbname = db
318     dbconn = DB(name="information_schema")
319     cursor = dbconn.execute("SELECT * FROM `SCHEMATA` WHERE `SCHEMA_NAME` = '%s'" % (dbname))
320     row = cursor.fetchone()
321     if row == None:
322         try:
323             file = config["rootpasswordfile"]
324             fd = open(file, "r")
325             password = fd.readline().rstrip()
326             fd.close()
327         except:
328             error('Unable to read mysql root password from "%s"' % (file))
329             raise
330         dbconn = DB(user="root", passwd=password, name="information_schema")
331         debug(req, "Creating database")
332         dbconn.execute("CREATE DATABASE IF NOT EXISTS `%s`" % (dbname))
333         dbconn.execute("GRANT ALL PRIVILEGES ON `%s` . * TO 'errorlog'@'%%'" % (dbname))
334         dbconn.execute("GRANT ALL PRIVILEGES ON `%s` . * TO 'errorlog'@'localhost'" % (dbname))
335
336         dbconn = DB(name=dbname)
337
338         dbconn.execute("""CREATE TABLE IF NOT EXISTS `errors`
339                           (`id` INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
340                            `program` VARCHAR(255),
341                            `version` VARCHAR(32),
342                            `exception` VARCHAR(255),
343                            `lineno` VARCHAR(32),
344                            `filename` VARCHAR(255),
345                            `hostname` VARCHAR(255),
346                            `time` VARCHAR(32),
347                            `username` VARCHAR(255),
348                            `uname` VARCHAR(255),
349                            `savefilename` VARCHAR(255),
350                            `cmdline` TEXT,
351                            `trace` TEXT); """)
352
353
354 def delete_db(req, db=config["dbname"]):
355
356     dbconn = None
357     try:
358         debug(req, "delete %s" % (db))
359         dbname = db
360         req.content_type = "text/plain"
361         if db != None:
362             try:
363                 file = "/root/mysql-root-pass.txt"
364                 fd = open(file, "r")
365                 password = fd.readline().rstrip()
366                 fd.close()
367             except:
368                 error('Unable to read mysql root password from "%s"' % (file))
369                 raise
370             dbconn = DB(name="information_schema")
371             cursor = dbconn.execute("SELECT * FROM `SCHEMATA` WHERE `SCHEMA_NAME` = '%s'" % (dbname))
372             row = cursor.fetchone()
373             if row != None:
374                 dbconn=DB(name=dbname, user="root", passwd=password)
375                 dbconn.execute("DROP DATABASE `%s`" % (dbname))
376             dbconn.close()
377         req.write("OK")
378         request = None
379     finally:
380         if dbconn != None:
381             dbconn.close()
382
383
384 def reset(req, db=config["dbname"]):
385     if db:
386         dbname = db
387         dbconn = DB(name=dbname)
388         dbconn.execute("TRUNCATE `nodes`")
389         dbconn.close()
390     req.write("OK");