#!/usr/bin/env jython

#
# The main loading program for ippToPsps. See usage below.
#

import time
import sys
import os
import logging.config

from ipptopsps import IppToPsps
from gpc1db import Gpc1Db
from datastore import Datastore
from dvoobjects import DvoObjects
from batch import Batch

'''
Queue class
'''
class Queue(IppToPsps):

    '''
    Constructor
    '''
    def __init__(self, argv):
        super(Queue, self).__init__(argv)

        # create various objects
        self.gpc1Db = Gpc1Db(self.logger, self.config)
        self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb)

        try:
            self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb)
        except:
            self.exitProgram("Unable to create instance of DvoObject")
            raise

        self.skychunk.printAll()

        if len(argv) > 2: self.parsePollTimeArg(sys.argv[2])

    '''
    Main processing loop.
    Tiled area is defined in 'skychunk' and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing
    by one or more loading clients that may be running on any host
    '''
    def run(self):

        # this outer while loop simply waits a few minutes then starts queuing again (as more stuff may appear in DVO over time)
        while True:

            self.ippToPspsDb.removeAllBoxes()
        
            # queue up batches that are processed but not loaded to datastore
            for batchType in self.skychunk.batchTypes: 

                self.logger.infoTitle("Previous failed datastore loads")
                self.publishAnyUnpublishedBatches(batchType)

                # get totals for whole area to check if there is anything to do
                processedIDs = self.ippToPspsDb.getProcessedIDs(batchType)
                consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType)

                # for object batches, get full list of stuff from Dvo
                # XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo
                if batchType == "OB":
                    self.dvoObjects.setSkyArea(
                            self.skychunk.minRa,
                            self.skychunk.maxRa,
                            self.skychunk.minDec,
                            self.skychunk.maxDec)
                    rows = self.dvoObjects.allPopulatedRegionInfo
                    self.dvoObjects.printSummary()

                # for other batches, info comes from gpc1 database
                else:
                    rows = self.gpc1Db.getItemsInThisDVODbForThisStage(
                            self.skychunk.dvoLabel, 
                            batchType, 
                            self.skychunk.minRa, 
                            self.skychunk.maxRa, 
                            self.skychunk.minDec, 
                            self.skychunk.maxDec)

                # EAM TEST I/O
                self.logger.infoPair("received rows from db:", len(rows))

                # first report total stuff
                allIDs = []
                for row in rows:
                    try: allIDs.append(row[0])
                    except: pass

                ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs))
                self.logger.infoPair("All unprocessed items", "%d" % len(ids))

                # now trim full list down to only pending stuff
                pending = [[], [], []]
                for row in rows:
                    try:
                        findId = row[0]
                    except: continue
                    if findId in ids: pending.append([row[0], row[1], row[2]])

                # store all items in temp table
                self.logger.infoPair("Inserting all pending items into", "ippToPsps database")
                self.ippToPspsDb.storeAllItems(pending)

                # loop through full range of RA/Dec queueing stuff in boxes of size self.skychunk.boxSize
                if len(ids) > 0:

                    self.logger.info("+-------------+-------------+-------------+-------------+")
                    self.logger.info("|     RA      |     Dec     |   box size  | unprocessed |")
                    self.logger.info("+-------------+-------------+-------------+-------------+")
    
                    # starting positions
                    ra = self.skychunk.minRa + self.skychunk.halfBox
                    dec = self.skychunk.minDec + self.skychunk.halfBox
    
                    while ra <= self.skychunk.maxRa:
                       while dec <= self.skychunk.maxDec:
            
                           box_id = self.ippToPspsDb.insertBox(ra, dec)
            
                           ids = self.ippToPspsDb.getItemsInThisThisBox(ra, dec)
    
                           self.logger.info("|    %5.1f    |    %5.1f    |  %9d  |  %9d  |" % (
                                       ra, 
                                       dec, 
                                       self.skychunk.boxSize, 
                                       len(ids)))
                
                           if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids)

                           dec = dec + self.skychunk.boxSize
                       dec = self.skychunk.minDec + self.skychunk.halfBox
                       ra = ra + self.skychunk.boxSize
            
                self.logger.info("+-------------+-------------+-------------+-------------+")

            self.checkClientStatus()
            if not self.waitForPollTime(): break

    '''
    Finds batches that processed ok, but did not get loaded to the datastore, then tries to load them
    '''
    def publishAnyUnpublishedBatches(self, batchType):
    
        batchIDs = self.ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(batchType)
        self.logger.infoPair("%s batches" % batchType, "%d" % len(batchIDs))
    
        for batchID in batchIDs:
    
            batchName = Batch.getNameFromID(batchID)
            subDir = Batch.getSubDir(self.skychunk.basePath, batchType, self.skychunk.dvoLabel)
            tarballFile = Batch.getTarballFile(batchID)
            self.logger.infoPair("Batch name", batchName)
    
            Batch.publishToDatastore(self.datastore, batchID, batchName, subDir, tarballFile)
   

    '''
    Overrides base-class version
    '''
    def printUsage(self):
        super(Queue, self).printUsage("[<repeat time in hours>]")


'''
Start of program.
'''
try:
    queue = Queue(sys.argv)
    queue.run()
    queue.exitProgram("completed")
except: pass

