Changeset 31403

Timestamp:

Apr 28, 2011, 3:55:01 PM (15 years ago)

Author:

rhenders

Message:

skipping missing smf files; can now use either full dvo tables, or on-the-fly ones; screening out objID duplicates using a UNIQUE column and UPDATE IGNORE

File:

• trunk/ippToPsps/jython/detectionbatch.py (modified) (16 diffs)

trunk/ippToPsps/jython/detectionbatch.py

@@ -1 +1 @@
 #!/usr/bin/env jython
 
+import os.path
+import sys
 import stilts
 from java.lang import *
@@ -17 +19 @@
     Constructor
     '''
-    def __init__(self, logger, camID, inputFile, test=False):
+    def __init__(self, logger, camID, inputFile, test=False, useFullTables=False):
        super(DetectionBatch, self).__init__(
                logger,
                "detection", 
                inputFile, 
-               "MD04") # TODO
+               "MD04",
+               useFullTables) # TODO
                #"3PI") # TODO
 
@@ -177 +180 @@
 
         tableName = "ImageMeta_" + ota
-        self.logger.info("   Dealing with table " + tableName)
         
         # drop then re-create table
@@ -321 +323 @@
 
         tableName = "Detection_" + ota
-        self.logger.info("   Dealing with table " + tableName)
         
         # drop then re-create table
@@ -396 +397 @@
 
         tableName = "SkinnyObject_" + ota
-        self.logger.info("   Dealing with table " + tableName)
         
         # drop then re-create table
@@ -425 +425 @@
 
         tableName = "ObjectCalColor_" + ota
-        self.logger.info("   Dealing with table " + tableName)
         
         # drop then re-create table
@@ -451 +450 @@
 
     '''
-    Applies indexes to the PSPS tables TODO need this?
-    '''
-    def indexPspsTables(self):
-
+    Applies indexes and other constraints to the PSPS tables
+    '''
+    def alterPspsTables(self):
+
+        self.logger.info("Altering PSPS tables")
         self.logger.info("Creating indexes on PSPS tables")
-#        self.createIndex("StackDetection", "ippDetectID")
- #       self.createIndex("StackApFlx", "ippDetectID")
-  #      self.createIndex("StackModelFit", "ippDetectID")
+        self.scratchDb.makeColumnUnique("Detection", "objID")
+        self.scratchDb.createIndex("Detection", "ippDetectID")
 
     '''
@@ -482 +481 @@
     Updates provided table with DVO IDs from DVO table
     '''
-    def updateDvoIDs(self, table, sourceID, imageID):
-
-        self.logger.info("Updating table '" + table + "' with DVO IDs...")
-        sql = "UPDATE " + table + " AS a, dvoDetection AS b SET \
+    def updateDvoIDs(self, table, sourceID, externID):
+
+        imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID)
+        self.logger.info("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
+        sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \
                a.ippObjID = b.ippObjID, \
                a.detectID = b.detectID, \
@@ -495 +495 @@
 
         self.scratchDb.stmt.execute(sql)
-        self.logger.info("...done")
 
 
@@ -527 +526 @@
 
                 # store sourceID/imageID combo in Db so DVO can look up later
-                self.scratchDb.insertNewDvoImage(header['SOURCEID'], header['IMAGEID'])
+                if not self.useFullTables:
+                    self.scratchDb.insertNewDvoImage(header['SOURCEID'], header['IMAGEID'])
 
                 # store these for later
@@ -538 +538 @@
              
         # now run DVO code to get all IDs
-        self.getIDsFromDVO()
+        if not self.useFullTables: self.getIDsFromDVO()
 
         # loop through all OTAs again to update with DVO IDs
@@ -555 +555 @@
 
                 ota = "XY%d%d" % (x, y)
+                self.logger.info("******************* Dealing with OTA " + ota + " *******************")
 
                 # populate remainder of tables
@@ -561 +562 @@
                 # now add DVO IDs
                 self.updateDvoIDs("Detection_" + ota, sourceIDs[ota], imageIDs[ota])
-
                 self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID")
                 self.updateImageID("Detection_" + ota, x, y) 
@@ -571 +571 @@
 
                 # update ImageMeta with count of detections for this OTA and photoCodeID
-                sql = "UPDATE ImageMeta_" + ota + " SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(header['SOURCEID'], header['IMAGEID']))
+                sql = "UPDATE ImageMeta_" + ota + " SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
                 self.scratchDb.stmt.execute(sql)
 
@@ -633 +633 @@
 gpc1Db = Gpc1Db(logger)
 camIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.V2", "cam")
-#camIDs = gpc1Db.getIDsInThisDVODbForThisStage("ThreePi.V1", "cam")
 
 logger.info("Found %d exposures" % len(camIDs))
@@ -640 +639 @@
 for camID in camIDs:
 
+    logger.info("-------------------------------------------------- cam ID: %d" % camID)
+
     file = gpc1Db.getCameraStageSmf(camID)
-    detectionBatch = DetectionBatch(logger, camID, file, False)
-
-    if detectionBatch.alreadyProcessed():
+    if not os.path.isfile(file):
+        logger.error("Cannot read file at '" + file)
+        continue
+
+    detectionBatch = DetectionBatch(logger, camID, file, False, True)
+
+    if not detectionBatch.alreadyProcessed():
 
         detectionBatch.createEmptyPspsTables()
-        #detectionBatch.importIppTables(".*.psf")
+        detectionBatch.importIppTables(".*.psf")
         if detectionBatch.populatePspsTables():
             detectionBatch.exportPspsTablesToFits("([a-zA-Z]+)")
             detectionBatch.writeBatchManifest()
-            detectionBatch.reportNullsInAllPspsTables(False)
-            detectionBatch.createTarball()
-            detectionBatch.publishToDatastore()
+            #detectionBatch.reportNullsInAllPspsTables(False)
+            #detectionBatch.createTarball()
+            #detectionBatch.publishToDatastore()
     
-    i = i+1
-    if i > 0: break
-
+            i = i+1
+           # if i > 0: sys.exit()
+