#!/usr/bin/env python

import sys
import os
import socket
import subprocess
import MySQLdb
import logging

###################################
# 
class Constants: # Constants and/or default values
    force = False # if we want to ignore the lock
    limit_undefined = 5000  # the maximum number of entries that we want to refresh
    limit = 1000  # the maximum number of entries that we want to refresh
    loggingLevel = logging.INFO # 
    skip_stages = []
    updatedb_binary = '/home/panstarrs/schastel/local/bin/updatedb'
    locate_binary = '/home/panstarrs/schastel/local/bin/locate'
    mlocate_db_template = '/export/%s.0/schastel/mlocate/%s.%s.mlocate_db'
    lock_directory = '/tmp/diskMonitoring'
    lock_filename_template = '%s/lock.%s'
    nebulous_directory_template = '/export/%s.0/nebulous/%s'
    update_table_undefined_python_script = '/home/panstarrs/schastel/dev/FilesMonitoring/update_table_undefined.py'
    update_table_python_script = '/home/panstarrs/schastel/dev/FilesMonitoring/update_table.py'

###################################
# 
class FilesDatabaseConnection:
    def __init__(self, nodename, dirname):
        self.nodename = nodename
        self.dirname = dirname
        if nodename == 'neverland':
            self.db = MySQLdb.connect('localhost', 'ipp', 'ipp', 'DisksMonitoring')
        else:
            self.db = MySQLdb.connect('localhost', 'root', '', 'DisksMonitoring')
        self.count_inserts = 0
    def close():
        self.db.commit()
        self.db.close()
    def find_instance(self, neb_ins_id, subdirname, filename):
        query = 'SELECT neb_ins_id FROM neb_0_%s WHERE neb_ins_id = %s' % (self.dirname,
                                                                           neb_ins_id)
        cursor = self.db.cursor()
        cursor.execute(query)
        rows = cursor.fetchall()
        if len(rows) == 0:
            print 'New entry'
        elif len(rows) == 1:
            print 'Existing entry'
        else:
            print 'Problem'
        cursor.close()
    def get_instance_ids(self):
        query = 'SELECT neb_ins_id, subdirname, filename FROM neb_0_%s' % (self.dirname)
        cursor = self.db.cursor()
        cursor.execute(query)
        ids = dict()
        for row in cursor.fetchall():
            ids[row[0]] = (row[1], row[2])
        cursor.close()
        return ids
    def add_new_file(self, filename, neb_ins_id, subdirname, debug = False):
        query = """INSERT INTO neb_0_%s (
  filename, 
  neb_ins_id, 
  subdirname, 
  nebkey, 
  creation_date)
VALUES (
  '%s', 
  %s, 
  '%s', 
  '%s', 
  NOW())""" % (self.dirname,
               filename,
               neb_ins_id,
               subdirname,
               generate_nebulous_key(filename))
        if debug:
            logger.debug('<sql>%s;</sql>' % query)
            return
        cursor = self.db.cursor()
        cursor.execute(query)
        self.count_inserts += 1
        cursor.close()
        if self.count_inserts >= 100:
            self.db.commit()
            self.count_inserts = 0

    def mark_as_deleted(self, neb_ins_id):
        statement = 'UPDATE neb_0_%s SET status=\'deleted\' WHERE neb_ins_id=%d' % (self.dirname,
                                                                                    neb_ins_id)
        logger.debug('... Statement: (%s)' % (statement))
        cursor = self.db.cursor()
        cursor.execute(statement)
        cursor.close()

    def optimize_table(self):
        query = 'OPTIMIZE TABLE neb_0_%s' % (self.dirname)
        logger.debug('... Running statement "%s"' % (query))
        cursor = self.db.cursor()
        cursor.execute(query)
        cursor.close()
        self.commit()
    def commit(self):
        self.db.commit()

###################################
# 
class SummaryDatabaseConnection:
    def __init__(self):
        self.nodename = nodename
        self.dirname = dirname
        if nodename == 'neverland':
            self.db = MySQLdb.connect('localhost', 'root', 'root', 'DiskMonitoringSummary')
        else:
            self.db = MySQLdb.connect('localhost', 'ipp', 'ipp', 'DiskMonitoringSummary')

###################################
# generate_nebulous_key()
def generate_nebulous_key(filename):
    elements = filename.split('.')
    neb_key = '.'.join(elements[1:]).replace(':', '/')
    return neb_key

###################################
# process_options()
def process_options(arguments):
    # Default values: see usage()
    force = Constants.force
    limit = Constants.limit
    skip_stages = Constants.skip_stages
    index = 1
    while index<len(arguments):
        if arguments[index] == '-h':
            usage()
            sys.exit(0)
        elif arguments[index] == '-f':
            force = True
            index += 1
        elif arguments[index] == '-l' or arguments[index] == '-limit':
            limit = int(arguments[index+1])
            index += 2
        elif arguments[index] == '-v':
            logger.setLevel(logging.DEBUG)
            index += 1
        elif arguments[index] == '-skip':
            skip_stages = arguments[index+1].split(',')
            logger.info('Stage(s) to skip: %s' % (arguments[index+1]))
            index += 2
        else:
            dirname = arguments[index]
            index += 1
    try:
        return (dirname, force, limit, skip_stages)
    except UnboundLocalError: # Variable dirname is not defined
        usage()
        sys.exit(1)

###################################
# usage()
def usage():
    """
Usage: %s [-f] [-v] [{-l|-limit} <limit>] <dirname>
       %s -h

    Refreshes files monitoring information about the
    /export/%s.0/nebulous/<dirname> nebulous storage 
    directory. The different stages are:
      1. Run updatedb on that directory
      2. Remove deleted files from the FilesMonitoring database
      3. Add new files in the FilesMonitoring database
      4. Refresh the status of the <limit> oldest entries in the corresponding 
         table.

  Arguments:
    <dirname>: The nebulous directory to be refreshed 00..ff

  Options:
    -f    Force to ignore the lock. Beware of multiple instances of this
          program running at the same time
    -h    Displays this help
    -v    Be verbose
    -l/-limit <limit>
          Update up to <limit> new and then oldest files in the monitoring
          database. By default, <limit> is %d

    -skip <stage#>[,<stage#>...]
          You are supposed to know what you are doing by using this
          option ! Debugging mode: Skip <stage#>
"""
    nodename = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0]
    sys.stderr.write(usage.__doc__ % (sys.argv[0], sys.argv[0],
                                      nodename, Constants.limit))

###################################
# Main
if __name__ == '__main__':
    #######################
    # 0. Initializations
    #######################
    # 0.1. Setup logging
    logger = logging.getLogger()
    formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', 
                                  '%Y-%m-%dT%H:%M:%S')
    logging_output = logging.StreamHandler(sys.stderr)
    logging_output.setFormatter(formatter)
    logger.addHandler(logging_output)
    logger.setLevel(Constants.loggingLevel)
    # 0.2. Get arguments
    (dirname, force, limit, skip_stages) = process_options(sys.argv)
    nodename = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0]
    logger.info('%s for [%s]/[%s]' % (sys.argv[0], nodename, dirname))
    logger.info('Stage 0 (cannot be skipped): initializations')
    # 0.3. Put a lock in /tmp/diskMonitoring/
    lock_directory = Constants.lock_directory
    lock_filename = Constants.lock_filename_template % (lock_directory, dirname)
    if not os.path.exists(lock_directory):
        os.mkdir(lock_directory)
    try:
        open(lock_filename)
        if not force:
            logger.error('Locked\n')
            sys.exit(2)
        logger.info('... Existing lock ignored')
    except IOError, e:
        open(lock_filename, 'w').close()
    # 0.4. Open a connection to the FilesMonitoring database
    filesMonitoring_database = FilesDatabaseConnection(nodename, dirname)
    logger.info('Stage 0 complete')

    #######################
    # 1. Stage 1: Run updatedb on directory /export/%s.0/nebulous/<dirname>
    #######################
    logger.info('Stage 1: updatedb')
    if '1' not in skip_stages:
        directory = Constants.nebulous_directory_template % (nodename, dirname)
        mlocate_db = Constants.mlocate_db_template % (nodename,
                                                      nodename, 
                                                      dirname)
        p = subprocess.Popen( [Constants.updatedb_binary,
                               '-l', '0',
                               '-U', directory,
                               '-o', mlocate_db],
                              stderr = subprocess.PIPE,
                              stdout = subprocess.PIPE)
        if p.wait()!=0:
            logger.error('Error while running updatedb')
            for line in p.stderr:
                logger.error(line)
            sys.exit(3)
        logger.info('... Stage 1 complete')
    else:
        logger.info('... Stage 1 skipped')

    #######################
    # Stage 1.9: Collect all ins_id from the database and the mlocate file
    #######################
    logger.info('Stage 1.9 (cannot be skipped): Get ins_id both from database and mlocate file')
    if '2' in skip_stages and '3' in skip_stages: #Ignore if there is no database update
        logger.info('... Stage 1.9 skipped since both 2 and 3 have to be skipped')
    else:
        # Collect ins_id from the database
        ins_ids_in_db = filesMonitoring_database.get_instance_ids()
        logger.debug('... Got %d entries from the database' % len(ins_ids_in_db))
        # Collect ins_id from the locate file
        mlocate_db = Constants.mlocate_db_template % (nodename,
                                                      nodename, 
                                                      dirname)
        p = subprocess.Popen( [Constants.locate_binary,
                               '-d', mlocate_db,
                               '.'],
                              stderr = subprocess.PIPE,
                              stdout = subprocess.PIPE)
        p.poll()
        count = 0
        acceptable = 0
        root = Constants.nebulous_directory_template % (nodename,
                                                        dirname)
        ins_ids_in_mlocate = dict()
        for line in p.stdout:
            long_filename = line[:-1].replace(root, 
                                              '')
            if long_filename.startswith('/'):
                long_filename = long_filename[1:]
            try:
                (subdirname, filename) = long_filename.split('/')
                if logger.isEnabledFor(logging.DEBUG):
                    logger.debug('... -> %s' % line[:-1])
                    logger.debug('...... [%s] split in (%s,%s)' % (long_filename,
                                                                     subdirname, 
                                                                     filename))
                fields = filename.split('.')
                if fields[0] != '':
                    ins_ids_in_mlocate[int(fields[0])] = (filename, subdirname)
                else:
                    logger.warn('... Check file [%s]' % line[:-1])
                acceptable += 1
            except ValueError:
                pass # These are the directories
            count += 1
        logger.debug('... Looked at %d entries in updatedb database' % count)
        logger.debug('... Found %d acceptable entries in updatedb database' % acceptable)
        if acceptable+257!=count:
            # We expect to have 256 subdirectories and the root dir
            # excluded, that is exactly count-257 acceptable entries
            # in the updatedb database. If not, it means that there is
            # an unexpected file in the root directory
            logger.debug('... !!! There might be a problem in updatedb database !!!')
        logger.debug('... Got %d entries from the mlocate file' % len(ins_ids_in_mlocate))
        # Differences
        logger.debug('... Checking differences')
        deleted_entries = dict()
        existing_entries = dict()
        new_entries = dict()
        for key in ins_ids_in_db.keys():
            if ins_ids_in_mlocate.has_key(key):
                existing_entries[key] = ins_ids_in_mlocate[key]
            else:
                if ins_ids_in_db.has_key(key):
                    logger.debug('Adding [%s, %s] to files to delete' % (ins_ids_in_db[key]))
                    deleted_entries[key] = ins_ids_in_db[key]
                else:
                    logger.debug('No entry in database for key [%s]' % key)
        for key in ins_ids_in_mlocate.keys():
            if ins_ids_in_db.has_key(key):
                existing_entries[key] = ins_ids_in_db[key]
            else:
                new_entries[key] = ins_ids_in_mlocate[key]
        logger.info('... New files.............: %d' % len(new_entries))
        logger.info('... Deleted files.........: %d' % len(deleted_entries))
        logger.info('... Already existing files: %d' % (len(existing_entries)))
        logger.info('... Stage 1.9 complete')

    #######################
    # 2. Stage 2: Remove deleted files from the FilesMonitoring database
    #######################
    logger.info('Stage 2: remove deleted files from the FilesMonitoring database')
    if '2' not in skip_stages:
        logger.info('... %d entries to delete' % len(deleted_entries))
        for neb_ins_id in deleted_entries.keys():
            filesMonitoring_database.mark_as_deleted(neb_ins_id)
        filesMonitoring_database.commit()
        logger.info('... Stage 2 complete')
    else:
        logger.info('... Stage 2 skipped')

    #######################
    # 3. Add new files to the FilesMonitoring database
    #######################
    logger.info('Stage 3: add new files to the FilesMonitoring database')
    if '3' not in skip_stages:
        logger.info('... %d entries to add' % len(new_entries))
        for neb_ins_id in new_entries.keys():
            filesMonitoring_database.add_new_file(new_entries[neb_ins_id][0], 
                                                  neb_ins_id, 
                                                  new_entries[neb_ins_id][1])
        filesMonitoring_database.commit()
        logger.info('... Stage 3 complete')
    else:
        logger.info('... Stage 3 skipped')

    #######################
    # 3.9. OPTIMIZE TABLE
    #######################
    logger.info('Stage 3.9 (cannot be skipped): optimize table')
    if '2' in skip_stages and '3' in skip_stages: #Ignore if no db update
        logger.info('... Stage 3.9 skipped since both 2 and 3 have to be skipped')
    else:
        filesMonitoring_database.optimize_table()
        logger.info('... Stage 3.9 complete')

    #######################
    # 4. Refresh the status of the <limit> oldest entries in the
    #    corresponding table
    #######################
    logger.info('Stage 4: refresh the status of the <limit> oldest entries (limit = %d)' % limit)
    if '4' not in skip_stages:
        p = subprocess.Popen( [Constants.update_table_undefined_python_script,
                               '-l', str(Constants.limit_undefined),
                               dirname],
                              stderr = subprocess.PIPE,
                              stdout = subprocess.PIPE)
        if p.wait()!=0:
            logger.error('Error while running %s' % Constants.update_table_undefined_python_script)
            for line in p.stderr:
                logger.error(line)
            sys.exit(3)
        p = subprocess.Popen( [Constants.update_table_python_script,
                               '-l', str(limit),
                               dirname],
                              stderr = subprocess.PIPE,
                              stdout = subprocess.PIPE)
        if p.wait()!=0:
            logger.error('Error while running %s' % Constants.update_table_python_script)
            for line in p.stderr:
                logger.error(line)
            sys.exit(3)
        logger.info('... Stage 4 complete')
    else:
        logger.info('... Stage 4 skipped')

    #######################
    # 5. Stage 5.9: Remove lock
    #######################
    logger.info('Stage 5.9 (cannot be skipped): Remove lock')
    logger.debug('... Removing %s' % lock_filename)
    os.unlink(lock_filename)
    logger.info('... Stage 5.9 complete')
