#!/usr/bin/env jython

#
# The main loading program for ippToPsps. See usage below.
#

import time
import sys
import os
import socket
import logging.config
from xml.etree.ElementTree import ElementTree, Element, tostring

from pslogger import PSLogger
from gpc1db import Gpc1Db
from ipptopspsdb import IppToPspsDb
from datastore import Datastore
from batch import Batch

from initbatch import InitBatch
from stackbatch import StackBatch
from detectionbatch import DetectionBatch

'''
Finds batches that processed ok, but did not get loaded to the datastore, then tries to load them
'''
def publishAnyUnpublishedBatches(batchType):

    batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(EPOCH, DVOGPC1LABEL, batchType)
    logger.infoPair("%s batches" % batchType, "%d" % len(batchIDs))

    for batchID in batchIDs:

        batchName = Batch.getNameFromID(batchID)
        subDir = Batch.getSubDir(BASEPATH, batchType, DVOGPC1LABEL)
        tarballFile = Batch.getTarballFile(batchID)
        logger.infoPair("Batch name", batchName)

        Batch.publishToDatastore(datastore, batchID, batchName, subDir, tarballFile)

'''
Queues up all available stuff for this batch type
'''
def queueItems(batchType):

    logger.infoTitle("Queuing " + batchType + " items")

    allIDs = []

    # see if there are any IDs listed in the config
    try:
        for element in configDoc.findall('options/ids/id'):
            allIDs.append(int(element.text))
    except:
        logger.errorPair("Problem loading IDs from config under", "options/ids/id") 

    if len(allIDs) > 0:
        logger.infoPair("Using config IDs (not DVO)", "%d" % len(allIDs))
    # no config dictated IDs, so get from DVO instead
    else:

        if MINRA and MAXRA and MINDEC and MAXDEC:
            allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType, MINRA, MAXRA, MINDEC, MAXDEC)
        else:
            allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType)

        logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs))

    # if in FORCE mode, then queue full list
    if FORCE: ids = allIDs

    # if not in TEST mode, then only queue un-processed items
    else:
        processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOGPC1LABEL)
        consistaentlyFailedIDs = ippToPspsDb.getConsistentlyFailedIDs(batchType, EPOCH, DVOGPC1LABEL)
        ids = list(set(allIDs) - set(processedIDs) - set(consistaentlyFailedIDs))
        logger.infoPair("Processed items", "%d" % len(processedIDs))
        logger.infoPair("Consistently failing items", "%d" % len(consistaentlyFailedIDs))
        logger.infoPair("Unprocessed items", "%d" % len(ids))

    logger.infoPair("%s items queued" % batchType, "%d" % len(ids))

    # loop round IDs of all items to be processed
    ippToPspsDb.lockBatchTable()
    for id in ids:

        batchID = ippToPspsDb.createNewBatch(
                batchType,
                id,
                SURVEY,
                EPOCH,
                DVOGPC1LABEL,
                datastore.product,
                FORCE)
        
        if batchID < 0: continue;

        ippToPspsDb.unlockTables()

        if batchType == "P2":
            batch = DetectionBatch(logger,
                    CONFIG,
                    configDoc,
                    gpc1Db,
                    ippToPspsDb,
                    id,
                    batchID)
        elif batchType == "ST":
            batch = StackBatch(logger,
                    CONFIG,
                    configDoc,
                    gpc1Db,
                    ippToPspsDb,
                    id,
                    batchID)

        batch.run()

        ippToPspsDb.lockBatchTable()

        logger.infoSeparator()

        # if in TEST mode, then quit after one batch
        if TEST: break

    ippToPspsDb.unlockTables()

'''
Start of program.
'''

if len(sys.argv) > 1: CONFIG = sys.argv[1]
else:
    print "** Usage: " + sys.argv[0] + " <configPath> [init]"
    sys.exit(1)

if len(sys.argv) > 2 and sys.argv[2] == "init":
    QUEUE_IN = 1
else:
    QUEUE_IN = 0

# open config file
configDoc = ElementTree(file=CONFIG)
TEST = int(configDoc.find("options/testMode").text)

# set up logging
logging.setLoggerClass(PSLogger)
logger = logging.getLogger("ippToPspsLog")

# get hostnamee and PID for unique log naming
HOST = socket.gethostname()
PID = os.getpid()

# if in test mode, print log to screen, otherwise, only to file
logger.setup(configDoc, "ippToPsps", TEST, not TEST, HOST, PID)

# create database objects
gpc1Db = Gpc1Db(logger, configDoc)
ippToPspsDb = IppToPspsDb(logger, configDoc)
datastore = Datastore(logger, configDoc, ippToPspsDb)

# check we connected ok
if not gpc1Db.everythingOK: sys.exit(1)
if not ippToPspsDb.everythingOK: sys.exit(1)

# get values from the configutaion file
POLLPERIOD = 600
DVOGPC1LABEL = configDoc.find("dvo/gpc1Label").text
FORCE = int(configDoc.find("options/force").text)
EPOCH = configDoc.find("options/epoch").text
PUBLISH = int(configDoc.find("options/publishToDatastore").text)
BASEPATH = configDoc.find("localOutPath").text
QUEUE_P2 = int(configDoc.find("options/queueP2").text)
QUEUE_ST = int(configDoc.find("options/queueST").text)
SURVEY = configDoc.find("options/survey").text

# get equatorial coord limits, if any
MINRA = None
MAXRA = None
MINDEC = None
MAXDEC = None
try:
    MINRA = float(configDoc.find("dvo/minRA").text)
    MAXRA = float(configDoc.find("dvo/maxRA").text)
    MINDEC = float(configDoc.find("dvo/minDec").text)
    MAXDEC = float(configDoc.find("dvo/maxDec").text)
except:
    pass

# prompt user: FORCE and PUBLISH is a dangerous combination
if FORCE and PUBLISH and not QUEUE_IN:
   response = raw_input("\n*** Are you sure you want to publish data with the 'force' option enabled (y/n)? ")
   if response != "y": sys.exit(1)

# prompt user: TEST and PUBLISH is a dangerous combination
if TEST and PUBLISH and not QUEUE_IN:
   response = raw_input("\n*** Are you sure you want to publish data with the 'testMode' option enabled (y/n)? ")
   if response != "y": sys.exit(1)

# print info to log
logger.infoSeparator()
logger.infoTitle("ippToPsps loading started")
logger.infoPair("Configuration file", CONFIG)
logger.infoPair("Loading epoch", EPOCH)
logger.infoPair("DVO gpc1 label", DVOGPC1LABEL)
logger.infoBool("Queuing P2 batches?", QUEUE_P2)
logger.infoBool("Queuing ST batches?", QUEUE_ST)
logger.infoBool("Forcing?", FORCE)
logger.infoBool("Test mode?", TEST)
logger.infoBool("Publishing?", PUBLISH)

'''
Main processing loop:

 1) creates an IN batch if requested
 2) queues available P2 batches (if requested)
 3) queues available ST batches (if requested)
 4) if in test mode, quits
 5) otherwise, waits then repeats from 2

'''
while True:

    # queue all items for all batch types, sequentially
    if QUEUE_IN:
       batchID = ippToPspsDb.createNewBatch(
                 "IN",
                 0,
                 SURVEY,
                 EPOCH,
                 DVOGPC1LABEL,
                 datastore.product,
                 1)
       if batchID > 0:
           batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb, batchID)
           batch.run()
       break;

    # queue up batches that are processed but not loaded to datastore
    logger.infoTitle("Previous failed datastore loads")
    publishAnyUnpublishedBatches("P2")
    publishAnyUnpublishedBatches("ST")

    if QUEUE_P2: queueItems("P2")
    if QUEUE_ST: queueItems("ST")

    # in the TEST mode, we quit after submitting one batch
    if TEST: break

    # wait for the POLLPERIOD before checking for new ids
    logger.infoPair("Finished. Waiting for", "%.1f minutes before checking DVO for new items" % (POLLPERIOD/60.0))
    time.sleep(POLLPERIOD)


