Changeset 32630 for branches/eam_branches/ipp-20110906/ippToPsps/jython/load.py

Timestamp:

Nov 8, 2011, 2:44:12 PM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20110906

Files:

• . (modified) (1 prop)
• ippToPsps/jython/load.py (modified) (9 diffs)

branches/eam_branches/ipp-20110906/ippToPsps/jython/load.py

@@ -7 +7 @@
 import time
 import sys
+import os
+import socket
 import logging.config
 from xml.etree.ElementTree import ElementTree, Element, tostring
@@ -35 +37 @@
         logger.infoPair("Batch name", batchName)
 
-        Batch.publishToDatastore(datastore, ippToPspsDb, batchID, batchName, subDir, tarballFile)
-
+        Batch.publishToDatastore(datastore, batchID, batchName, subDir, tarballFile)
 
 '''
@@ -54 +55 @@
         logger.errorPair("Problem loading IDs from config under", "options/ids/id") 
 
-    # no config dictated IDs, so get from DVO instead
     if len(allIDs) > 0:
         logger.infoPair("Using config IDs (not DVO)", "%d" % len(allIDs))
+    # no config dictated IDs, so get from DVO instead
     else:
-        allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType)
+
+        if MINRA and MAXRA and MINDEC and MAXDEC:
+            allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType, MINRA, MAXRA, MINDEC, MAXDEC)
+        else:
+            allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType)
+
         logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs))
 
@@ -67 +73 @@
     else:
         processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOGPC1LABEL)
-        logger.infoPair("Processed %s items" % batchType, "%d" % len(processedIDs))
-        ids = list(set(allIDs) - set(processedIDs))
-        logger.infoPair("Unprocessed %s items" % batchType, "%d" % len(ids))
+        consistaentlyFailedIDs = ippToPspsDb.getConsistentlyFailedIDs(batchType, EPOCH, DVOGPC1LABEL)
+        ids = list(set(allIDs) - set(processedIDs) - set(consistaentlyFailedIDs))
+        logger.infoPair("Processed items", "%d" % len(processedIDs))
+        logger.infoPair("Consistently failing items", "%d" % len(consistaentlyFailedIDs))
+        logger.infoPair("Unprocessed items", "%d" % len(ids))
 
     logger.infoPair("%s items queued" % batchType, "%d" % len(ids))
 
     # loop round IDs of all items to be processed
+    ippToPspsDb.lockBatchTable()
     for id in ids:
 
-        if not FORCE and ippToPspsDb.alreadyProcessed(batchType, id, EPOCH, DVOGPC1LABEL): continue
-        if not FORCE and ippToPspsDb.consistentlyFailed(batchType, id, EPOCH, DVOGPC1LABEL, 3): continue
+        batchID = ippToPspsDb.createNewBatch(
+                batchType,
+                id,
+                SURVEY,
+                EPOCH,
+                DVOGPC1LABEL,
+                datastore.product,
+                FORCE)
+        
+        if batchID < 0: continue;
+
+        ippToPspsDb.unlockTables()
 
         if batchType == "P2":
@@ -85 +104 @@
                     gpc1Db,
                     ippToPspsDb,
-                    id)
+                    id,
+                    batchID)
         elif batchType == "ST":
             batch = StackBatch(logger,
@@ -92 +112 @@
                     gpc1Db,
                     ippToPspsDb,
-                    id)
+                    id,
+                    batchID)
 
         batch.run()
+
+        ippToPspsDb.lockBatchTable()
+
+        logger.infoSeparator()
 
         # if in TEST mode, then quit after one batch
         if TEST: break
 
+    ippToPspsDb.unlockTables()
 
 '''
@@ -122 +148 @@
 logger = logging.getLogger("ippToPspsLog")
 
+# get hostnamee and PID for unique log naming
+HOST = socket.gethostname()
+PID = os.getpid()
+
 # if in test mode, print log to screen, otherwise, only to file
-logger.setup(configDoc, "ippToPsps", TEST, not TEST)
+logger.setup(configDoc, "ippToPsps", TEST, not TEST, HOST, PID)
 
 # create database objects
 gpc1Db = Gpc1Db(logger, configDoc)
 ippToPspsDb = IppToPspsDb(logger, configDoc)
-datastore = Datastore(logger, configDoc)
+datastore = Datastore(logger, configDoc, ippToPspsDb)
 
 # check we connected ok
@@ -143 +173 @@
 QUEUE_P2 = int(configDoc.find("options/queueP2").text)
 QUEUE_ST = int(configDoc.find("options/queueST").text)
+SURVEY = configDoc.find("options/survey").text
+
+# get equatorial coord limits, if any
+MINRA = None
+MAXRA = None
+MINDEC = None
+MAXDEC = None
+try:
+    MINRA = float(configDoc.find("dvo/minRA").text)
+    MAXRA = float(configDoc.find("dvo/maxRA").text)
+    MINDEC = float(configDoc.find("dvo/minDec").text)
+    MAXDEC = float(configDoc.find("dvo/maxDec").text)
+except:
+    pass
 
 # prompt user: FORCE and PUBLISH is a dangerous combination
@@ -180 +224 @@
     # queue all items for all batch types, sequentially
     if QUEUE_IN:
-       batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb)
-       batch.run()
+       batchID = ippToPspsDb.createNewBatch(
+                 "IN",
+                 0,
+                 SURVEY,
+                 EPOCH,
+                 DVOGPC1LABEL,
+                 datastore.product,
+                 1)
+       if batchID > 0:
+           batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb, batchID)
+           batch.run()
        break;