#!/usr/bin/env jython

#
# The main loading program for ippToPsps. See usage below.
#

import time
import sys
import os
import traceback
import logging.config

from ipptopsps import IppToPsps
from scratchdb import ScratchDb
from gpc1db import Gpc1Db
from datastore import Datastore
from batch import Batch
from dvodetections import DvoDetections

from initbatch import InitBatch
from stackbatch import StackBatch
from detectionbatch import DetectionBatch
from objectbatch import ObjectBatch

'''
Loader class
'''
class Loader(IppToPsps):

    '''
    Constructor
    '''
    def __init__(self, argv):
        super(Loader, self).__init__(argv, 1, 1)

        # create gpc1 database objects
        self.gpc1Db = Gpc1Db(self.logger, self.config)

        # connect to scratch database
        scratchDbs = ['ipptopsps_scratch', 'ipptopsps_scratch2', 'ipptopsps_scratch3']
        for dbName in scratchDbs:
            self.scratchDb = ScratchDb(self.logger, self.config, dbName)
            if self.scratchDb.anyOtherConnections():
                self.logger.errorPair("This scratch Db is already in use", dbName)
                self.scratchDb.disconnect()
                continue
            break

        if not self.scratchDb.connected:
            self.exitProgram("Cannot connect to a scratch database")
            raise
       
        # create Datastore objects
        self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)

        # if an IN batch is requested, create and quit
        if len(sys.argv) > 2 and sys.argv[2] == "init":
           batchID = self.ippToPspsDb.createNewBatch("IN", 0)
           if batchID > 0:
               batch = InitBatch(self.logger, 
                       self.config, 
                       self.gpc1Db, 
                       self.ippToPspsDb, 
                       self.scratchDb, 
                       batchID)
               batch.run()
           else:
               self.logger.errorPair("Could not create batch ID", "%d" % batchID)
               self.exitProgram("failed to create init batch")
               
           self.exitProgram("init batch created")
    

        # set a poll time of about 1 minute
        self.parsePollTimeArg("0.0166")

        self.config.printAll()

    '''
    Overrides base-class version so we can ensure our scratch Db is using the right DVO Db
    '''
    def refreshConfig(self):

        ret = super(Loader, self).refreshConfig()
        try: self.scratchDb
        except: return ret
           
        self.dvoDetections = DvoDetections(self.logger, self.config, self.scratchDb.dbName)

        return ret

    '''
    Overrides base_class version so we can break out of processing loops if the config changes
    '''
    def checkClientStatus(self):

        oldConfigName = self.config.name
        super(Loader, self).checkClientStatus()
        if oldConfigName != self.config.name:
            self.logger.infoPair("Config changed",  "from '" + oldConfigName + "' to '" + self.config.name + "'")
            self.config.printAll()
            return False

        return True

    '''
    Main processing loop
    '''
    def run(self):

        if not self.scratchDb.connected: return

        numAttempts = 0
        while True:

            abort = False
            for batchType in self.config.batchTypes:

                # get a stripe's worth of box IDs
                boxIds = self.ippToPspsDb.getStripeBoxIds(self.HOST, self.PID, batchType)

                numAttempts += 1

                self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
    
                if len(boxIds) > 0: numAttempts = 0

                for boxId in boxIds:
          
                    if not self.checkClientStatus():
                        abort = True
                        break
    
                    # get box info. if boxes have changed, break and start again
                    try:
                        boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
                    except:
                        self.logger.infoPair("Can't get details for this box", "%d" % boxId)
                        break
     
                    ids = self.ippToPspsDb.getPendingIdsForThisBox(boxId, batchType)
                
                    if len(ids) < 1: 
                        self.logger.debugPair("No " + batchType + " items found in this box", "skipping")
                        continue
                
                    self.logger.infoSeparator()
                    self.logger.infoTitle("New box")
                    self.logger.infoPair("Box dimensions", "%.1f / %.1f / %.1f" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE']))
                    self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids))
                    boxSizeWithBorder = boxDim['SIDE'] + (self.config.BORDER * 2)
     
                    useFullTables = 0
                    if batchType != "OB":
                        # look in DVO for this box (with extra border)
                        self.logger.infoPair("Querying DVO for this sky area", "")
                        self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
                        #self.dvoDetections.setSkyArea()
                        sizeToBeIngested = self.dvoDetections.getDiskSizeOfRegionsToBeIngested()
                        if sizeToBeIngested == 0.0: smfsPerGB = 999999999
                        else: smfsPerGB = len(ids)/sizeToBeIngested
                        self.logger.infoPair("DVO to be ingested", "%.1f GB" % sizeToBeIngested)
                        self.logger.infoPair("smfs-per-GB", "%.1f" % smfsPerGB)
                        # should do we pre-ingest stuff from DVO?
                        if batchType == 'P2' and smfsPerGB > 30:
                            if not self.dvoDetections.sync():
                                self.logger.errorPair("Could not sync DVO with MySQL", "skipping")
                                continue
     
                            useFullTables = 1
                
                    self.logger.infoBool("Using pre-ingested DVO data?", useFullTables)
                    if not self.processTheseItems(batchType, ids, useFullTables):
                        abort = True
                        break

            if abort or not self.checkClientStatus(): abort = True
            elif numAttempts > 1 and not self.waitForPollTime():  break

        
    '''
    Actually loops through items and processes them
    '''
    def processTheseItems(self, batchType, ids, useFullTables):
    
        self.logger.infoPair("%s items queued" % batchType, "%d" % len(ids))
    
        # loop round IDs of all items to be processed
        self.ippToPspsDb.lockBatchTable()
        unattemptedCount = 0
        for id in ids:
    
            batchID = self.ippToPspsDb.createNewBatch(batchType, id)
            
            if batchID < 0: 
                unattemptedCount += 1
                continue
    
            self.ippToPspsDb.unlockTables()
            self.ippToPspsDb.deletePendingItem(batchType, id)

            # catch any raised exceptions in batch constructors
            try:
                if batchType == "P2":
                    batch = DetectionBatch(self.logger,
                            self.config,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID,
                            useFullTables)
                elif batchType == "ST":
                    batch = StackBatch(self.logger,
                            self.config,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID,
                            useFullTables)
    
                elif batchType == "OB":
                    batch = ObjectBatch(self.logger,
                            self.config,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID,
                            useFullTables)
    
                batch.run()
   
            # if batch fails, ignore and move on to the next one
            except: 
                self.logger.errorPair("Problem with this %s batch (%d)" % (batchType, id), "skipping")
                pass

            if not self.checkClientStatus(): return False
    
            self.ippToPspsDb.lockBatchTable()
    
            self.logger.infoSeparator()
    
        self.ippToPspsDb.unlockTables()
        self.logger.infoPair("Unattempted batches", "%d" % unattemptedCount)

        return True
    
    '''
    Overrides base-class version
    '''
    def printUsage(self):
        super(Cleanup, self).printUsage("<[init]>")

    
'''
Start of program.
'''
try:
    loader = Loader(sys.argv)
    loader.run()
    loader.exitProgram("completed")
except Exception, e:
    print str(e)
    traceback.print_exc()

