Changeset 30662

Timestamp:

Feb 17, 2011, 8:53:01 AM (15 years ago)

Author:

Serge CHASTEL

Message:

Automated measurements of Nebulous activity (experimental)

Location:

trunk/tools/log_parsing

Files:

• nebulous_activity.py (added)
• parse_apache_log.py (modified) (4 diffs)

trunk/tools/log_parsing/parse_apache_log.py

@@ -4 +4 @@
 import re
 import datetime
+import fileinput
 
 nebulousPattern = re.compile('POST /nebulous HTTP/1.1')
+errorPattern = re.compile('exit signal Segmentation fault')
+
 # Resolution: 10 minutes of 60 seconds of 1000000 microseconds
 datetime.resolution = 10*60*1000000
 
 def message(size):
+    """
+    Guess the message type according to the message size
+    """
     if size == 488:
         # delete == 488
@@ -34 +40 @@
 
 def show_results(counts, output=sys.stderr):
+    """
+    Display results
+    """
     output.write('#             Epoch | Create | FindIn | Delete | FailSt |   Stat |  Unkn. | NonNeb\n')
     for count in sorted(counts.iterkeys()):
@@ -45 +54 @@
                                                                            counts[count][6]))
 
-if __name__ == '__main__':
+def process(linesInFile = None):
+    """
+    Process Nebulous entries in apache2 log file
+    """
     counts = dict()
-
     linesCount = 1
-    for line in sys.stdin:
-        if linesCount % 10000 == 0:
-            sys.stderr.write('Analyzed lines: %d\n' % linesCount)
-            if linesCount % 100000 == 0:
+    if linesInFile == None:
+        input = fileinput.input()
+    else:
+        input = linesInFile
+    for line in input:
+        if linesCount % 100000 == 0:
+            sys.stderr.write('\tAnalyzed lines: %d\n' % linesCount)
+            if linesCount % 500000 == 0:
                 show_results(counts)
         linesCount += 1
@@ -85 +100 @@
                 counts[dt.isoformat()] = [0, 0, 0, 0, 0, 0, 0]
                 counts[dt.isoformat()][6] += 1
+    return counts
 
-    show_results(counts, sys.stdout)
+def show_errors(counts, output=sys.stderr):
+    """
+    Display error results
+    """
+    output.write('#             Epoch | Errors\n')
+    for count in sorted(counts.iterkeys()):
+        output.write('%19s | %6d\n' % (count, counts[count]))
+
+def processErrorLog(linesInFile = None):
+    """
+    Process errors in apache2 error log file
+    """
+    counts = dict()
+    linesCount = 1
+    if linesInFile == None:
+        input = fileinput.input()
+    else:
+        input = linesInFile
+    for line in input:
+        if linesCount % 100000 == 0:
+            sys.stderr.write('\tAnalyzed lines: %d\n' % linesCount)
+            if linesCount % 500000 == 0:
+                show_errors(counts)
+        linesCount += 1
+        if errorPattern.search(line):
+            elements = line.split(']')
+            date = elements[0]
+            date = date[1:]
+            # print '[%s]' % (date)
+            dt = datetime.datetime.strptime(date, 
+                                            '%a %b %d %H:%M:%S %Y')
+            dt = dt.replace(minute = int(dt.minute/10)*10,
+                            second = 0)
+            # print dt.isoformat()
+            try:
+                counts[dt.isoformat()] += 1
+            except KeyError:
+                counts[dt.isoformat()] = 1
+    return counts
+
+def usage():
+    sys.stderr.write(main.__doc__)
+
+def main(arguments):
+    """
+Usage: parse_apache_log [-a <nebulous apache access_log file>] [-e <nebulous apache error_log file>] 
+"""
+    mainName = arguments.pop()
+    if len(arguments) == 0:
+        sys.stderr.write('Reading from /dev/stdin\n')
+        counts = process()
+        show_results(counts, sys.stdout)
+    elif len(arguments) % 2 != 0:
+        sys.stderr.write('Arguments length: %d\n' % (len(arguments)))
+        usage()
+        sys.exit(1)
+    else:
+        accessFilenames = []
+        errorFilenames = []
+        while len(arguments) != 0:
+            flag = arguments.pop()
+            filename = arguments.pop()
+            if flag == '-a':
+                accessFilenames.append(filename)
+            elif flag == '-e':
+                errorFilenames.append(filename)
+            else:
+                sys.stderr.write('Unknown flag: [%s]' % (flag))
+                usage()
+                sys.exit(1)
+        for accessFilename in accessFilenames:
+            sys.stderr.write('Processing: [%s]' % (accessFilename))
+            inFile = open(accessFilename)
+            counts = process(inFile)
+            show_results(counts, sys.stdout)
+        for errorFilename in errorFilenames:
+            counts = processErrorLog(errorFilename)
+            show_errors(counts, sys.stdout)
+
+if __name__ == '__main__':
+    main(sys.argv)