#!/usr/bin/env python

import sys
import MySQLdb
import subprocess
import _mysql_exceptions
import re

toGigs = 1024.**3
toGigsFromK = 1024.**2

def process_arguments(argv):
    if len(argv) == 1:
        usage()
        sys.exit(1)
    elif argv[1] == 'all':
        nodesFilename = '/home/panstarrs/schastel/local/share/ippMhpccStorageNodes'
        nodesFile = open(nodesFilename)
        nodes = []
        for line in nodesFile:
            nodes.append(line[:-1])
    elif argv[1] == 'set0':
        nodes = [ "ipp004", "ipp005", "ipp006", "ipp007", "ipp008", "ipp009", 
                  "ipp060", "ipp061", "ipp062", "ipp063", "ipp064", "ipp065", "ipp066", ]
    elif argv[1] == 'set1':
        nodes = [ "ipp010", "ipp011", "ipp012", "ipp013", "ipp014", 
                  "ipp015", "ipp016", "ipp017", "ipp018", "ipp019", ]
    elif argv[1] == 'set2':
        nodes = [ "ipp020", "ipp021", "ipp023", "ipp024", 
                  "ipp025", "ipp026", "ipp027", "ipp028", "ipp029", ]
    elif argv[1] == 'set3':
        nodes = [ "ipp030", "ipp031", "ipp032", "ipp033", "ipp034", 
                  "ipp035", "ipp036", "ipp037", "ipp038", "ipp039", ]
    elif argv[1] == 'set4':
        nodes = [ "ipp040", "ipp041", "ipp042", "ipp043", "ipp044", 
                  "ipp045", "ipp046", "ipp047", "ipp048", "ipp049", ]
    elif argv[1] == 'set5':
        nodes = [ "ipp050", "ipp051", "ipp052", "ipp053", "ipp054", 
                  "ipp055", "ipp056", "ipp057", "ipp058", "ipp059", ]
    else:
        nodes = argv[1:]
    return nodes

def get_nebdf_output():
    nebdf_output = dict()
    p = subprocess.Popen( ['neb-df'],
                          stderr = subprocess.PIPE,
                          stdout = subprocess.PIPE)
    if p.wait()!=0:
        logger.error('Error while running neb-df')
        for line in p.stderr:
            logger.error(line)
        sys.exit(3)
    for line in p.stdout:
        if not line.startswith('Filesystem') and not line.startswith('/summmary') and not line.startswith(' '):
            line = re.sub('\s+', ':', line[:-1])
            (node, available, used, free, percentage, mountpoint) = line.split(':')
            nebdf_output[node] = (float(available)/toGigsFromK, float(used)/toGigsFromK, float(free)/toGigsFromK)
    return nebdf_output

def is_mysql_server_running(node):
    try:
        MySQLdb.connect(node, 'ippuser', 'ipp', 'DisksMonitoring')
        return "mysql server: Running"
    except _mysql_exceptions.OperationalError:
        return "mysql server: Stopped"

def is_update_running(node):
    p = subprocess.Popen( ['/home/panstarrs/schastel/dev/FilesMonitoring/Supervision/node_status.py', node],
                          stderr = subprocess.PIPE,
                          stdout = subprocess.PIPE)
    if p.wait()!=0:
        return "Can't tell if node's running update/refresh (ssh error)"
    for line in p.stdout:
        return " ".join(line[:-1].split(' ')[1:])

if __name__ == '__main__':
    nodes = process_arguments(sys.argv)
    nebdf = get_nebdf_output()
    dms_db = MySQLdb.connect('ippc11', 'ippuser', 'ipp',
                             'DiskMonitoringSummary')
    dms_cursor = dms_db.cursor()
    aggregate_by_role = dict()
    aggregate_by_stage = dict()
    nodes_sizes = dict()
    aggregate = dict()
    raw_aggregate = dict()
    for node in nodes:
        try:
            query = "SELECT SUM(counts) FROM CurrentDiskUsage WHERE node='%s' AND status!='deleted'" % node
            dms_cursor.execute(query)
            total = dms_cursor.fetchall()[0][0]
            #print total, query
            query = "SELECT SUM(counts) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted'" % node
            dms_cursor.execute(query)
            defined = dms_cursor.fetchall()[0][0]
            #print defined, query
            percentage_done = float(defined) / float(total)
            #
            try:
                query = "SELECT SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted'" % node
                dms_cursor.execute(query)
                nodes_sizes[node] = float(dms_cursor.fetchall()[0][0]) / percentage_done / toGigs
            except TypeError, e:
                print e
            print 'Node %s: %5.1f%% done | %20s | %40s' % (node, percentage_done * 100., 
                                                           is_mysql_server_running(node),
                                                           is_update_running(node))
            query = "SELECT role, SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted' GROUP BY role" % node
            dms_cursor.execute(query)
            for row in dms_cursor.fetchall():
                if row[0] != 'undefined':
                    try:
                        aggregate_by_role[row[0]] += (float(row[1])/percentage_done/toGigs)
                    except KeyError:
                        aggregate_by_role[row[0]] = (float(row[1])/percentage_done/toGigs)
            query = "SELECT stage, SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted' GROUP BY stage" % node
            dms_cursor.execute(query)
            for row in dms_cursor.fetchall():
                if row[0] != 'undefined':
                    try:
                        aggregate_by_stage[row[0]] += (float(row[1])/percentage_done/toGigs)
                    except KeyError:
                        aggregate_by_stage[row[0]] = (float(row[1])/percentage_done/toGigs)
            query = "SELECT stage, role, SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted' GROUP BY stage, role" % node
            dms_cursor.execute(query)
            for row in dms_cursor.fetchall():
                if row[0] != 'undefined':
                    try:
                        aggregate[row[0]][row[1]] += (float(row[2])/percentage_done/toGigs)
                        raw_aggregate[row[0]][row[1]] += float(row[2])/toGigs
                    except KeyError:
                        try:
                            aggregate[row[0]][row[1]] = (float(row[2])/percentage_done/toGigs)
                            raw_aggregate[row[0]][row[1]] = float(row[2])/toGigs
                        except KeyError:
                            aggregate[row[0]] = dict()
                            raw_aggregate[row[0]] = dict()
                            aggregate[row[0]][row[1]] = (float(row[2])/percentage_done/toGigs)
                            raw_aggregate[row[0]][row[1]] = float(row[2])/toGigs
        except TypeError:
            print 'Node %s: being refreshed' % node

    print
    print "### By role (in gigs / relative % / in teras)"
    total = 0.
    for size in aggregate_by_role.itervalues():
        total += size
    for role, size in aggregate_by_role.iteritems():
        print "%10s: %10.1f %10.1f%% %10.1f" % (role, size, size/total*100., size/1024.)
    print "%10s: %10.1f %11s %10.1f" % ("Total", total, "", total/1024.)

    print
    print "### By stage (in gigs / relative % / in teras)"
    total = 0.
    for size in aggregate_by_stage.itervalues():
        total += size
    for stage, size in aggregate_by_stage.iteritems():
        print "%10s: %10.1f %10.1f%% %10.1f" % (stage, size, size/total*100., size/1024.)
    print "%10s: %10.1f %11s %10.1f" % ("Total", total, "", total/1024.)
    print

    print
    print "### Corrected Data (in gigs)"
    header = '%10s' % ''
    roles = dict()
    for stage in aggregate.iterkeys():
        header = '%s%10s' % (header, stage)
        for role in aggregate[stage].iterkeys():
            roles[role] = ''
    print header
    for role in roles.iterkeys():
        line = '%10s' % role
        for stage in aggregate.iterkeys():
            try:
                line = '%10s%10.1f' % (line, aggregate[stage][role])
            except KeyError:
                line = '%10s%10.1f' % (line, 0.0)
        print line

    print
    print "### Raw Data (in gigs)"
    header = '%10s' % ''
    roles = dict()
    for stage in aggregate.iterkeys():
        header = '%s%10s' % (header, stage)
        for role in aggregate[stage].iterkeys():
            roles[role] = ''
    print header
    for role in roles.iterkeys():
        line = '%10s' % role
        for stage in aggregate.iterkeys():
            try:
                line = '%10s%10.1f' % (line, raw_aggregate[stage][role])
            except KeyError:
                line = '%10s%10.1f' % (line, 0.0)
        print line

    print
    print "### By node: Mine \t neb-df \tDifference (compared to disk size)\t\"Non-nebulous\" space"
    neb_total = 0.
    nebdf_total = 0.
    nonneb_total = 0.
    available = 0.
    for node in nodes:
        try:
            neb_total += nodes_sizes[node]
            nebdf_total += nebdf[node][1]
            nonneb_total += (nebdf[node][1]-nodes_sizes[node])
            available += nebdf[node][0]
            print '%10s: %10.2f %10.2f %10.2f%% %10.2f' % ( node, nodes_sizes[node], nebdf[node][1], 
                                                            (nodes_sizes[node]-nebdf[node][1])/nebdf[node][0]*100.,
                                                            nebdf[node][1]-nodes_sizes[node]
                                                            )
        except KeyError:
            print '%10s: Incomplete data' % node
    print "---------------------------------------------------"
    print '%10s: %10.2f %10.2f %10.2f%% %10.2f' % ( "Total", neb_total, nebdf_total, 
                                                    (neb_total-nebdf_total)/available*100.,
                                                    nonneb_total )
    exclude_nodes = ['ipp004', 'ipp005', ]
    for node in exclude_nodes:
        neb_total -= nodes_sizes[node]
        nebdf_total -= nebdf[node][1]
        nonneb_total -= (nebdf[node][1]-nodes_sizes[node])
        available -= nebdf[node][0]
    print '%10s: %10.2f %10.2f %10.2f%% %10.2f' % ( "Total*", neb_total, nebdf_total, 
                                                    (neb_total-nebdf_total)/available*100.,
                                                    nonneb_total )
    print "Total* = Total excluding nodes:", exclude_nodes
    
