Changeset 31504 for trunk/ippToPsps/jython/detectionbatch.py

Timestamp:

May 10, 2011, 9:40:49 AM (15 years ago)

Author:

rhenders

Message:

using base-class run() methid; overriden versions of importIppTables() and exportPspsTablesToFits(); reworked alreadyProcessed() method;

File:

• trunk/ippToPsps/jython/detectionbatch.py (modified) (22 diffs)

trunk/ippToPsps/jython/detectionbatch.py

@@ -6 +6 @@
 from java.lang import *
 from java.sql import *
+
 from batch import Batch
 from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
 
 import logging.config
@@ -19 +22 @@
     Constructor
     '''
-    def __init__(self, logger, camID, inputFile, test=False, useFullTables=False):
+    def __init__(self, 
+                 logger,
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 camID, 
+                 inputFile, 
+                 test=False, 
+                 useFullTables=False):
+
        super(DetectionBatch, self).__init__(
                logger,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               camID,
                "detection", 
                inputFile, 
-               "MD04",
-               useFullTables) # TODO
+               "MD04", # TODO
                #"3PI") # TODO
-
-       self.logger.info("DetectionBatch constructor. Creating batch from: '" + inputFile + "'")
-
-       meta = self.gpc1Db.getCameraStageMeta(camID)
-      
-       self.expID = meta[0];
-       self.expName = meta[1];
-       self.distGroup = meta[2];
-
-       self.logger.info("Processing exposure with ID: %d, name: %s and distribution group: %s" % (self.expID, self.expName, self.distGroup))
+               useFullTables)
+
+       if not self.everythingOK: return
+
+       # meta data to the log
+       self.logger.info("New Detection Batch:")
+       self.logger.info("Cam ID:             %d" % self.id)
+       self.logger.info("file:               %s" % inputFile)
+       self.logger.info("Exp ID:             %d" % self.expID)
+       self.logger.info("Exp name:           %s" % self.expName)
+       self.logger.info("Distribution group: %s" % self.distGroup)
 
        # create an output filename, which is {expID}.FITS
@@ -167 +183 @@
         ," + self.header['PCA2X0Y2'] + " \
         )"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID))
@@ -184 +200 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE ImageMeta"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -310 +326 @@
                )"
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         self.scratchDb.updateFilterID(tableName, self.filter)
         self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
-        self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
+        if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
         self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0")
 
@@ -327 +343 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE Detection"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -375 +391 @@
                ,EXT_NSIGMA \
                FROM " + ota + "_psf"
-
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         # set obsTime
         sql = "UPDATE " + tableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
         self.scratchDb.updateAllRows(tableName, "historyModNum", "0")
@@ -387 +402 @@
         self.scratchDb.updateFilterID(tableName, self.filter)
 
-        # now delete bad flux
+        # now delete bad flux and bad chip positions
         self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "instFlux")
         self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "peakADU")
@@ -401 +416 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE SkinnyObject"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -415 +430 @@
                ,surveyID \
                FROM Detection_" + ota
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
@@ -429 +444 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE ObjectCalColor"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -443 +458 @@
                ,filterID \
                FROM Detection_" + ota
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
@@ -484 +499 @@
 
         imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID)
-        self.logger.info("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
+        self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
         sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \
                a.ippObjID = b.ippObjID, \
@@ -494 +509 @@
                AND b.imageID = " + str(imageID)
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
 
@@ -511 +526 @@
 
         # loop through all OTAs and populate ImageMeta extensions
+        self.logger.info("Reading all fits headers and populating ImageMeta tables")
         for x in range(self.startX, self.endX):
             for y in range(self.startY, self.endY):
@@ -524 +540 @@
                 # load corresponding header into memory
                 header = self.findAndReadFITSHeader(ota + ".hdr", file)
+                if not header:
+                    self.logger.error("No header found for OTA " + ota)
+                    continue
+
 
                 # store sourceID/imageID combo in Db so DVO can look up later
@@ -578 +598 @@
 
                 # update ImageMeta with count of detections for this OTA and photoCodeID
-                sql = "UPDATE ImageMeta_" + ota + " SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
-                self.scratchDb.stmt.execute(sql)
+                sql = "UPDATE ImageMeta_" + ota + " \
+                       SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
+                self.scratchDb.execute(sql)
 
                 self.populateSkinnyObjectTable(ota)
@@ -603 +624 @@
         # update FrameMeta with count OTAs in this file and total number of photometric reference sources
         sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         
         return True
@@ -613 +634 @@
 
         sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y) 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -619 +640 @@
     '''
     def alreadyProcessed(self):
+
+        meta = self.gpc1Db.getCameraStageMeta(self.id)
+        self.expID = meta[0];
+        self.expName = meta[1];
+        self.distGroup = meta[2];
 
         return self.ippToPspsDb.alreadyProcessed("detection", "exp_id", self.expID)
@@ -634 +660 @@
 
 
+    '''
+    Overriding this method. Filter to only import *.psf extensions
+    '''
+    def importIppTables(self, filter=""):
+       return super(DetectionBatch, self).importIppTables(".*.psf")
+
+
+    '''
+    Overriding this method. Use regex to trim off, eg _XY33 extension
+    '''
+    def exportPspsTablesToFits(self, regex="(.*)"):
+       return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")
+
+
+# TODO put in config
+useFullTables=True
+testMode=False
+
 logging.config.fileConfig("logging.conf")
 logger = logging.getLogger("detectionbatch")
+logger.setLevel(logging.INFO)
 logger.info("Starting")
 
 gpc1Db = Gpc1Db(logger)
+ippToPspsDb = IppToPspsDb(logger)
+scratchDb = ScratchDb(logger, useFullTables)
+
 camIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.V2", "cam")
 logger.info("Found %d exposures" % len(camIDs))
@@ -645 +693 @@
 for camID in camIDs:
 
-    logger.info("-------------------------------------------------- cam ID: %d" % camID)
+    #if camID < 43764: continue # TODO 
 
     file = gpc1Db.getCameraStageSmf(camID)
-    if not os.path.isfile(file):
-        logger.error("Cannot read file at '" + file)
-        continue
-
-    detectionBatch = DetectionBatch(logger, camID, file, False, True)
-
-    if not detectionBatch.alreadyProcessed():
-
-        detectionBatch.createEmptyPspsTables()
-        detectionBatch.importIppTables(".*.psf")
-        if detectionBatch.populatePspsTables():
-            detectionBatch.exportPspsTablesToFits("([a-zA-Z]+)")
-            detectionBatch.writeBatchManifest()
-            #detectionBatch.reportNullsInAllPspsTables(False)
-            #detectionBatch.createTarball()
-            #detectionBatch.publishToDatastore()
-    
-            i = i+1
-           # if i > 0: sys.exit()
-
+
+    detectionBatch = DetectionBatch(logger,
+                                    gpc1Db,
+                                    ippToPspsDb,
+                                    scratchDb,
+                                    camID, 
+                                    file, 
+                                    testMode, 
+                                    useFullTables)
+    detectionBatch.run()
+