Changeset 31402 for trunk/ippToPsps/jython/stackbatch.py

Timestamp:

Apr 28, 2011, 3:52:48 PM (15 years ago)

Author:

rhenders

Message:

skipping missing cmf files; hardcoded stuff to access nightly or deep stacks for MD04; code to swicth between 'full' dvo tables, or on-the-fly; adding uniquness to objID tabkle due to duplicates coming out of IPP; getting meta data from GPC1 rather than cmf headers as formats are inconststent

File:

• trunk/ippToPsps/jython/stackbatch.py (modified) (19 diffs)

trunk/ippToPsps/jython/stackbatch.py

@@ -1 +1 @@
 #!/usr/bin/env jython
+
+import os.path
+import sys
 
 import stilts
@@ -17 +20 @@
     Constructor
     '''
-    def __init__(self, logger, skyID, inputFile):
+    def __init__(self, logger, skyID, inputFile, stackType, useFullTables=False):
        super(StackBatch, self).__init__(
                logger,
                "stack", 
                inputFile, 
-               "MD04") # TODO
+               "MD04",
+               useFullTables) # TODO
 
        self.logger.info("StackBatch constructor. Creating batch from: '" + inputFile + "'")
 
        self.skyID = skyID
+
+       # get filterID using init table
+       self.filter = self.header['FPA.FILTER']
+       self.filter = self.filter[0:1]
+
+       self.stackType = stackType
+       meta = self.gpc1Db.getStackStageMeta(self.skyID, self.header['FPA.FILTER'])
+       if len(meta) < 1: return
+       self.stackID = meta[0];
+       self.skycell = meta[1];
+
+       # determine skycell from header value
+       #self.skycell = "skycell.34" #= self.header['SKYCELL']
+       self.skycell = self.skycell[8:]
+
+       self.logger.info("Processing stack with ID: %d, type: %s and skycell: %s filter: %s" % (self.stackID, self.stackType, self.skycell, self.filter))
+
 
        # delete PSPS tables
@@ -43 +64 @@
        #self.scratchDb.dropTable("SkyChip_xrad")
 
+       self.logger.info("Stack type: " + self.safeDictionaryAccess(self.header, self.stackType))
        # obs time makes no sense except for nightly stacks
-       self.logger.info("Stack type: " + self.header['STK_TYPE'])
-       if self.header['STK_TYPE'] != "NIGHTLY_STACK": self.header['MJD-OBS'] = "-999"
-
-       # determine skycell from header value
-       self.skycell = self.header['SKYCELL']
-       self.skycell = self.skycell[8:]
-
-       # get filterID using init table
-       self.filter = self.header['FPA.FILTER']
-       self.filter = self.filter[0:1]
+       #if self.header['STK_TYPE'] != "NIGHTLY_STACK": self.header['MJD-OBS'] = "-999"
 
        # create an output filename, which is {filterID}{skycellID}.FITS
@@ -64 +77 @@
 
        # insert what we know about this stack batch into the stack table
-       self.ippToPspsDb.insertStackMeta(self.batchID, self.skyID, self.header['STK_ID'], self.filter, self.header['STK_TYPE'])
+       self.ippToPspsDb.insertStackMeta(self.batchID, self.skyID, self.stackID, self.filter, self.stackType)
 
        # insert sourceID/imageID combo so DVO can look it up
-       self.scratchDb.insertNewDvoImage(self.header['SOURCEID'], self.header['IMAGEID'])
+       if not self.useFullTables:
+           self.scratchDb.insertNewDvoImage(self.header['SOURCEID'], self.header['IMAGEID'])
 
     '''
@@ -74 +88 @@
     def updateStackMetaID(self, table):
 
-        sql = "UPDATE " + table + "  SET stackMetaID=" + self.header['STK_ID']
+        sql = "UPDATE " + table + "  SET stackMetaID=" + str(self.stackID)
         self.scratchDb.stmt.execute(sql)
 
@@ -82 +96 @@
     def updateStackTypeID(self, table):
 
-        sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '"+self.header['STK_TYPE']+"'"
+        sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '" + self.stackType + "'"
         self.scratchDb.stmt.execute(sql)
 
@@ -195 +209 @@
             "+modelLong+"Covar78=b.EXT_COVAR_06_07,  \
             "+modelLong+"Covar88=b.EXT_COVAR_07_07,   \
-            "+modelLong+"serNu=b.EXT_PAR_07,   \
-            "+modelLong+"serNuErr=SQRT(EXT_COVAR_07_07)  \
+            "+"serNu=b.EXT_PAR_07,   \
+            "+"serNuErr=SQRT(EXT_COVAR_07_07)  \
             WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'"
 
@@ -214 +228 @@
         ,photoZero \
         ,expTime \
-        ,nP2Images \
         ,psfModelID \
         ,ctype1 \
@@ -229 +242 @@
         ,pc002002 \
          ) VALUES ( \
-        " + self.header['STK_ID'] + " \
+        " + str(self.stackID) + " \
         ," + self.skycell + " \
         ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \
         ," + self.header['FPA.ZP'] + " \
         ," + self.header['EXPTIME'] + " \
-        ," + self.header['NINPUTS'] + " \
-        ,'" + self.header['PSFMODEL'] + "' \
+        ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \
         ,'" + self.header['CTYPE1'] + "' \
         ,'" + self.header['CTYPE2'] + "' \
@@ -439 +451 @@
         self.logger.info("Procesing StackToImage table")
 
-        imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.header['STK_ID'])
+        imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.stackID)
 
         for imageID in imageIDs:
             sql = "INSERT INTO StackToImage (stackMetaID, imageID) \
                    VALUES (\
-                   " + self.header['STK_ID'] + ", " + imageID + ")"
+                   " + str(self.stackID) + ", " + imageID + ")"
             self.scratchDb.stmt.execute(sql)
+
+        # now update StackMeta with correct number of inputs
+        sql = "UPDATE StackMeta SET nP2Images = (SELECT COUNT(*) FROM StackToImage)"
+        self.scratchDb.stmt.execute(sql)
 
     '''
@@ -489 +505 @@
     Applies indexes to the PSPS tables
     '''
-    def indexPspsTables(self):
-
-        self.logger.info("Creating indexes on PSPS tables")
+    def alterPspsTables(self):
+
+        self.logger.info("Altering PSPS tables")
+        self.scratchDb.makeColumnUnique("StackDetection", "objID")
         self.scratchDb.createIndex("StackDetection", "ippDetectID")
         self.scratchDb.createIndex("StackApFlx", "ippDetectID")
@@ -512 +529 @@
     def updateDvoIDs(self, table):
 
+        imageID = self.scratchDb.getImageIDFromExternID(self.header['SOURCEID'], self.header['IMAGEID'])
+
         self.logger.info("Updating table '" + table + "' with DVO IDs...")
-        sql = "UPDATE " + table + " AS a, dvoDetection AS b SET \
+        sql = "UPDATE IGNORE " + table + " AS a, dvoDetectionFull AS b SET \
                a.ippObjID = b.ippObjID, \
                a.stackDetectID = b.detectID, \
@@ -519 +538 @@
                WHERE a.ippDetectID = b.ippDetectID \
                AND b.sourceID = " + self.header['SOURCEID'] + "\
-               AND b.imageID = " + self.header['IMAGEID']
-        self.scratchDb.stmt.execute(sql)
-        self.logger.info("...done")
+               AND b.imageID = " + str(imageID)
+        self.scratchDb.stmt.execute(sql)
 
 
@@ -529 +547 @@
     def populatePspsTables(self):
 
-        if not self.getIDsFromDVO(): return False
+        if not self.useFullTables:
+            if not self.getIDsFromDVO():
+                return False
 
         self.populateStackMeta()
@@ -548 +568 @@
     def alreadyProcessed(self):
 
-        return self.ippToPspsDb.alreadyProcessed("stack", "sky_id", self.skyID)
-
-
+        return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID)
 
 logging.config.fileConfig("logging.conf")
@@ -556 +574 @@
 logger.info("Starting")
 gpc1Db = Gpc1Db(logger)
+stackType = "NIGHTLY_STACK"
 skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky")
+#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky")
+#stackType = "DEEP_STACK"
 #skyIDs = [689]
 #skyIDs = [299]
-skyIDs = [306]
+#skyIDs = [302]
+#skyIDs = [8508]
 i = 0
 for skyID in skyIDs:
 
-    logger.info(" sky ID = %d" % skyID)
+    logger.info("-------------------------------------------------- sky ID: %d" % skyID)
 
     cmfFiles = gpc1Db.getStackStageCmfs(skyID)
@@ -569 +591 @@
     for file in cmfFiles:
 
-        logger.info("-------------- New Batch ----------")
-        stackBatch = StackBatch(logger, skyID, file)
-
-        if  stackBatch.alreadyProcessed():
+        if not os.path.isfile(file):
+            logger.error("Cannot read file at '" + file)
+            continue
+
+        stackBatch = StackBatch(logger, skyID, file, stackType, True)
+
+        if not stackBatch.alreadyProcessed():
 
             stackBatch.createEmptyPspsTables()
@@ -578 +603 @@
             if stackBatch.populatePspsTables():
  
-                stackBatch.reportNullsInAllPspsTables(False)
+                #stackBatch.reportNullsInAllPspsTables(False)
                 stackBatch.exportPspsTablesToFits()
                 stackBatch.writeBatchManifest()
@@ -584 +609 @@
                 #stackBatch.publishToDatastore()
 
-        i = i + 1
-        if i > 0: break # TODO just doing one filter for now
+                i = i + 1
+                #if i > 0: sys.exit()
 
 logger.info("Finished")