#!/usr/bin/env jython

#
# The main loading program for ippToPsps. See usage below.
#

import time
import sys
import os
import traceback
import logging.config

from ipptopsps import IppToPsps
from scratchdb import ScratchDb
from gpc1db import Gpc1Db
from datastore import Datastore
from batch import Batch
from dvodetections import DvoDetections

from initbatch import InitBatch
from stackbatch import StackBatch
from detectionbatch import DetectionBatch
from objectbatch import ObjectBatch

'''
Loader class
'''
class Loader(IppToPsps):

    '''
    Constructor
    '''
    def __init__(self, argv):
        super(Loader, self).__init__(argv, 1, 1)

        if self.skychunk.parallel:
            print "PARALLEL dvo"
        else: 
            print "SERIAL dvo"

        # connect to the gpc1 database
        self.gpc1Db = Gpc1Db(self.logger, self.config)

        # connect to scratch database
        scratchDbs = ['1', '2', '3']
        for dbVersion in scratchDbs:
            self.scratchDb = ScratchDb(self.logger, self.config, dbVersion)
            if not self.config.test and self.scratchDb.anyOtherConnections():
                self.logger.errorPair("This scratch Db is already in use", self.scratchDb.dbName)
                self.scratchDb.disconnect()
                continue
            break

        if not self.scratchDb.connected:
            self.exitProgram("Cannot connect to a scratch database")
            raise
       
        # create Datastore objects
        self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb)

        # if an IN batch is requested, create and quit
        if len(sys.argv) > 2 and sys.argv[2] == "init":
           batchID = self.ippToPspsDb.createNewBatch("IN", 0)
           if batchID > 0:
               batch = InitBatch(self.logger, 
                       self.config, 
                       self.skychunk, 
                       self.gpc1Db, 
                       self.ippToPspsDb, 
                       self.scratchDb, 
                       batchID)
               batch.run()
           else:
               self.logger.errorPair("Could not create batch ID", "%d" % batchID)
               self.exitProgram("failed to create init batch")
               
           self.exitProgram("init batch created")
    

        # if the 'once' option is passed, we do not loop multiple times in 'run'
        self.onePassOnly = 0
        if len(sys.argv) > 2 and sys.argv[2] == "once":
           self.onePassOnly = 1

        # set a poll time of about 1 minute
        self.parsePollTimeArg("0.0166")

        self.skychunk.printAll()

    '''
    Overrides base-class version so we can ensure our scratch Db is using the right DVO Db
    '''
    def refreshSkychunk(self):

        ret = super(Loader, self).refreshSkychunk()
        try: self.scratchDb
        except: return ret
           
        self.dvoDetections = DvoDetections(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb.dbName)

        return ret

    '''
    Overrides base_class version so we can break out of processing loops if the skychunk changes
    '''
    def checkClientStatus(self):

        oldSkychunkName = self.skychunk.name
        super(Loader, self).checkClientStatus()
        if oldSkychunkName != self.skychunk.name:
            self.logger.infoPair("Skychunk changed",  "from '" + oldSkychunkName + "' to '" + self.skychunk.name + "'")
            self.skychunk.printAll()
            return False

        return True

    '''
    Main processing loop
    '''
    def run(self):

        if not self.scratchDb.connected: return

        numAttempts = 0
        while True:

            abort = False
            for batchType in self.skychunk.batchTypes:

                # get a stripe's worth of box IDs
                boxIds = self.ippToPspsDb.getStripeBoxIds(self.HOST, self.PID, batchType)

                numAttempts += 1

                self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
                self.logger.infoPair("numAttempts", str(numAttempts))
                if len(boxIds) > 0: numAttempts = 0

                for boxId in boxIds:
                    self.logger.infoPair("Check","client status")
                    if not self.checkClientStatus():
                        abort = True
                        break
                    self.logger.infoPair("client status", "ok")
                    # get box info. if boxes have changed, break and start again
                    try:
                        boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
                    except:
                        self.logger.infoPair("Can't get details for this box", "%d" % boxId)
                        break
                    self.logger.infoPair("client status", "ok")
                    ids = self.ippToPspsDb.getPendingIdsForThisBox(boxId, batchType)
                
                    # ids are the stage_ids for items to be processed (eg, stack_id for a stack CMF, cam_id for P2 smf)

                    if len(ids) < 1: 
                        self.logger.debugPair("No " + batchType + " items found in this box", "skipping")
                        continue
                
                    self.logger.infoSeparator()
                    self.logger.infoTitle("New box")
                    self.logger.infoPair("Box dimensions", "%.1f / %.1f / %.1f" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE']))
                    self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids))
                    boxSizeWithBorder = boxDim['SIDE'] + (self.skychunk.BORDER * 2)
                    boxSizeSansBorder = boxDim['SIDE']
                    self.logger.infoPair("got here", "ok")
                    useFullTables = 0
                    if batchType != "OB":

                        if (self.dvoDetections.useStilts):
                            # look in DVO for this box (with extra border)
                            self.logger.infoPair("Querying DVO for this sky area", "")
                            self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
                            #self.dvoDetections.setSkyArea()
                            sizeToBeIngested = self.dvoDetections.getDiskSizeOfRegionsToBeIngested()
                            if sizeToBeIngested == 0.0: smfsPerGB = 999999999
                            else: smfsPerGB = len(ids)/sizeToBeIngested
                            self.logger.infoPair("DVO to be ingested", "%7.1e GB" % sizeToBeIngested)
                            self.logger.infoPair("smfs-per-GB", "%.1f" % smfsPerGB)

                            # should do we pre-ingest stuff from DVO?
                            # NOTE EAM : this skychunk loads the dvo detections into the mysql db
                            # XXXX EAM : this should happen for both P2 and Stack detection
                            # XXXX EAM : in parallel model, this should happen for all P2 & Stack batches
                            # 
                            if (batchType == 'P2' or batchType == 'ST') and smfsPerGB > 30:
                                if not self.dvoDetections.sync():
                                    self.logger.errorPair("Could not sync DVO with MySQL", "skipping")
                                    continue
                                
                                useFullTables = 1
                        else:
                            # XXX EAM : this is not currently an
                            # option. either remove the non-native
                            # ingest code or make it an option
                            # if we are using the native loader, always use it
                            # need to work out a good boundary / region strategy in coordination with 
                            # impact of mysql insertion
                            if not self.ippToPspsDb.isBoxIngested(boxId):
                                self.dvoDetections.nativeIngestDetections(boxId, boxDim['RA'], boxDim['DEC'], boxSizeSansBorder)

                            useFullTables = 1
                
                    ## else:
                    ##     # if we are using the native loader, always use it
                    ##     # need to work out a good boundary / region strategy in coordination with 
                    ##     # impact of mysql insertion
                    ##     self.dvoObjects.nativeIngestObjects(boxDim['RA'], boxDim['DEC'], boxSizeSansBorder)
                    ##     useFullTables = 1

                    '''
                    NOTE EAM : "TheseItems" refers to each of the stack CMFs, camera SMFs, or object batches
                    '''
                    self.logger.infoBool("Using pre-ingested DVO data?", useFullTables)
                    if not self.processTheseItems(batchType, ids, useFullTables):
                        abort = True
                        break
                    self.logger.infoPair("processed","ok")

            if abort or not self.checkClientStatus(): abort = True
            elif numAttempts > 1 and not self.waitForPollTime():  break

            if self.onePassOnly: self.exitProgram("one pass completed")
        
    '''
    Actually loops through items and processes them
    '''
    def processTheseItems(self, batchType, ids, useFullTables):
    
        self.logger.infoPair("%s items queued" % batchType, "%d" % len(ids))
    
        # loop round IDs of all items to be processed
        self.ippToPspsDb.lockBatchTable()
        self.logger.infoPair("heather:","lockbatchtable")
        unattemptedCount = 0
        for id in ids:
            self.logger.infoPair("heather:","in ids")
            self.logger.infoPair("heather:id",str(id))
            batchID = self.ippToPspsDb.createNewBatch(batchType, id)
            self.logger.infoPair("heather:","creatednewbatch")
            self.logger.infoPair("heather:batchId", str(batchID))
            if batchID < 0: 
                unattemptedCount += 1
                continue
            self.logger.infoPair("heather:","passed logic")
            self.ippToPspsDb.unlockTables()
            self.logger.infoPair("heather:","unlock table")
            self.ippToPspsDb.deletePendingItem(batchType, id)
            self.logger.infoPair("heather:","deleted pending item")

            # catch any raised exceptions in batch constructors
            try:
                if batchType == "P2":
                    batch = DetectionBatch(self.logger,
                            self.config,
                            self.skychunk,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID,
                            useFullTables)
                elif batchType == "ST":
                    batch = StackBatch(self.logger,
                            self.config,
                            self.skychunk,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID,
                            useFullTables)
    
                elif batchType == "OB":
                    batch = ObjectBatch(self.logger,
                            self.config,
                            self.skychunk,
                            self.gpc1Db,
                            self.ippToPspsDb,
                            self.scratchDb,
                            id,
                            batchID,
                            useFullTables)
    
                batch.run()
   
            # if batch fails, ignore and move on to the next one
            except Exception, e: 
                self.logger.error("EXCEPTION: " + str(e))
                self.logger.errorPair("Problem with this %s batch (%d)" % (batchType, id), "skipping")
                pass

            if not self.checkClientStatus(): return False
            self.logger.infoPair("checkclient status", "ok")

            self.ippToPspsDb.lockBatchTable()
            self.logger.infoPair("lockbatchtable", "ok")
            self.logger.infoSeparator()
    
        self.ippToPspsDb.unlockTables()
        self.logger.infoPair("Unattempted batches", "%d" % unattemptedCount)

        return True
    
    '''
    Overrides base-class version
    '''
    def printUsage(self):
        super(Loader, self).printUsage("<[init]>")

    
'''
Start of program.
'''
try:
    loader = Loader(sys.argv)
    loader.run()
    loader.exitProgram("completed")
except Exception, e:
    print str(e)
    traceback.print_exc()

