#!/usr/bin/env jython

#
# The main loading program for ippToPsps. See usage below.
#

import time
import sys
import os
import math
import logging.config

from ipptopsps import IppToPsps
from gpc1db import Gpc1Db
from datastore import Datastore
from dvoobjects import DvoObjects
from batch import Batch
from scratchdb import ScratchDb

'''
Queue class
'''
class Queue(IppToPsps):

    '''
    Constructor
    '''
    def __init__(self, argv):
        super(Queue, self).__init__(argv)

       # create various objects
        self.gpc1Db = Gpc1Db(self.logger, self.config)

        self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb)

        # connect to scratch database
        self.scratchDb = ScratchDb(self.logger, self.config)

        try:
            self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb, self.gpc1Db)
        except:
            self.exitProgram("Unable to create instance of DvoObject")
            raise

        self.skychunk.printAll()

        if len(argv) > 2: self.parsePollTimeArg(sys.argv[2])

    '''
    Main processing loop.
    Tiled area is defined in 'skychunk' and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing
    by one or more loading clients that may be running on any host
    '''
    def run(self):
        self.logger.info("------------- starting queue.run -----------------")
        # this outer while loop simply waits a few minutes then starts queuing again (as more stuff may appear in DVO over time)
        while True:

            self.ippToPspsDb.removeAllBoxes()

            # queue up batches that are processed but not loaded to datastore
            for batchType in self.skychunk.batchTypes: 
                self.logger.infoPair("BatchType",batchType)
                #self.logger.infoTitle("Previous failed datastore loads")
                #self.publishAnyUnpublishedBatches(batchType)

                # get totals for whole area to check if there is anything to do
                processedIDs = self.ippToPspsDb.getProcessedIDs(batchType)
                consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType)

                # for object batches, get full list of stuff from Dvo
                # XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo

                # for object-like batches, info comes from dvo
                if (batchType == "OB" or batchType =="DO" or batchType =="FO" or batchType == "FG" or batchType == "GO"):
                    self.dvoObjects.setSkyArea(
                            self.skychunk.minRa,
                            self.skychunk.maxRa,
                            self.skychunk.minDec,
                            self.skychunk.maxDec)
                    rows = self.dvoObjects.allPopulatedRegionInfo
                    self.dvoObjects.printSummary()

                # for other batches, info comes from gpc1 database
                else:
                    rows = self.gpc1Db.getItemsInThisDVODbForThisStage(
                            self.skychunk.dvoLabel, 
                            batchType, 
                            self.skychunk.minRa, 
                            self.skychunk.maxRa, 
                            self.skychunk.minDec, 
                            self.skychunk.maxDec,
                            self.skychunk.trange_start,
                            self.skychunk.trange_end)

                # EAM TEST I/O
                self.logger.infoPair("received rows from db:", len(rows))

                # first report total stuff
                allIDs = []
                for row in rows:
                    try: allIDs.append(row[0])
                    except: pass

                ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs))
                self.logger.infoPair("All unprocessed items", "%d" % len(ids))

                # now trim full list down to only pending stuff
                pending = [[], [], []]
                for row in rows:
                    try:
                        findId = row[0]
                    except: continue
                    if findId in ids: pending.append([row[0], row[1], row[2]])

                # store all items in temp table
                self.logger.infoPair("Inserting all pending items into", "ippToPsps database")
                self.ippToPspsDb.storeAllItems(pending)

                # loop through full range of RA/Dec queueing stuff in boxes of size self.skychunk.boxSize
                if len(ids) > 0:

                    self.logger.info("+-------------+-------------+-------------+-------------+")
                    self.logger.info("|     RA      |     Dec     |   box size  | unprocessed |")
                    self.logger.info("+-------------+-------------+-------------+-------------+")
    
                    # starting positions

                    dD = 0.5*self.skychunk.boxSize
                    
                    dec = self.skychunk.minDec + dD
                    #dec = self.skychunk.minDec

                    #if (dec > 85): raise
                    # this may be a problem for poles # HAF XXX
                    ####if (dec > 90): raise
                    # print "dec, dD: ", dec, dD ,self.skychunk.maxDec

                    # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
                    #dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
                    dR = 0.5*self.skychunk.boxSize
                                        
                    ra = self.skychunk.minRa + dR
                    
                    # print "ra, dR, maxRa", ra, dR, self.skychunk.maxRa

                    # XXX not sure why this is not done with a for-loop...
                    # answer: because python does not have a good float-style for-loop
                    while dec <= self.skychunk.maxDec:
                        while ra <= self.skychunk.maxRa:
                           # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
                           minRA  = ra  - dR
                           maxRA  = ra  + dR
                           minDEC = dec - dD
                           maxDEC = dec + dD

                           # this function inserts the box with given center and (linear) skychunk.boxSize 
                           box_id = self.ippToPspsDb.insertBox(ra, dec)
            
                           ids = self.ippToPspsDb.getItemsInThisBox(minRA, maxRA, minDEC, maxDEC)

                           self.logger.info("|    %5.1f    |    %5.1f    |  %9d  |  %9d  |" % (
                                       ra, 
                                       dec, 
                                       self.skychunk.boxSize, 
                                       len(ids)))
                
                           if len(ids) > 0: 
                               ## for myID in ids:
                               ##     print "id: %d : %10.6f - %10.6f | %10.6f - %10.6f" % (myID, minRA, maxRA, minDEC, maxDEC)

                               self.ippToPspsDb.insertPending(box_id, batchType, ids)
                               # print "inserted pending"

                           ra = ra + 2*dR
                           # print "new ra: ", ra

                        dec = dec + 2*dD
                        #if (dec > 85): raise
                        ###if (dec > 90): raise
                       #dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
                        dR = 0.5*self.skychunk.boxSize
                        ra = self.skychunk.minRa + dR

                self.logger.info("+-------------+-------------+-------------+-------------+")

            self.checkClientStatus()
            if not self.waitForPollTime(): break

    '''
    Finds batches that processed ok, but did not get loaded to the datastore, then tries to load them
    '''
    def publishAnyUnpublishedBatches(self, batchType):
    
        batchIDs = self.ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(batchType)
        self.logger.infoPair("%s batches" % batchType, "%d" % len(batchIDs))
    
        for batchID in batchIDs:
    
            batchName = Batch.getNameFromID(batchID)
            subDir = Batch.getSubDir(self.skychunk.basePath, batchType, self.skychunk.dvoLabel)
            tarballFile = Batch.getTarballFile(batchID)
            self.logger.infoPair("Batch name", batchName)
    
            Batch.publishToDatastore(self.datastore, batchID, batchName, subDir, tarballFile)
   

    '''
    Overrides base-class version
    '''
    def printUsage(self):
        super(Queue, self).printUsage("[<repeat time in hours>]")


'''
Start of program.
'''
try:
    queue = Queue(sys.argv)
    queue.run()
    queue.exitProgram("completed")
except: pass

