Changeset 31847 for trunk/ippToPsps/jython/stackbatch.py

Timestamp:

Jul 8, 2011, 2:59:10 PM (15 years ago)

Author:

rhenders

Message:

removed redundant alreadyProcessed() method; using new logging method; no longer storing stack_id locally, as this is 'id' field in super-class; deleting IPP tables prior to loading new ones in case some are not overwritten and then used to populate the wrong batch; skycell format change

File:

• trunk/ippToPsps/jython/stackbatch.py (modified) (21 diffs)

trunk/ippToPsps/jython/stackbatch.py

@@ -30 +30 @@
                  gpc1Db,
                  ippToPspsDb,
-                 skyID, 
-                 inputFile, 
-                 stackType):
+                 stackID):
 
        super(StackBatch, self).__init__(
@@ -39 +37 @@
                gpc1Db,
                ippToPspsDb,
-               skyID,
+               stackID,
                "stack", 
-               inputFile, 
-               "MD04") # TODO
+               gpc1Db.getStackStageCmf(stackID), 
+               "3PI") # TODO
 
        if not self.everythingOK: return
 
-       self.expTime = gpc1Db.getStackExpTime(self.stackID)
-
-       self.logger.info("got exp time of %d" % self.expTime)
+       self.stackType = "DEEP_STACK" # TODO
+
+       # get stack meta data
+       meta = self.gpc1Db.getStackStageMeta(self.id)
+       self.filter = meta[0];
+       self.filter = self.filter[0:1]
+       self.skycell = meta[1];
+       self.skycell = self.skycell[8:]
+       # mangling e.g. 0683.043 into 0683043 for now until we have a schema change
+       self.skycell = self.skycell.replace(".", "")
+       self.analysisVer = meta[2];
+
+       self.expTime = gpc1Db.getStackExpTime(self.id)
+
+       self.logger.debug("Got exp time of %d" % self.expTime)
 
        # delete PSPS tables
@@ -59 +69 @@
        self.scratchDb.dropTable("ObjectCalColor")
 
+       # delete IPP tables
+       self.scratchDb.dropTable("SkyChip_psf")
+       self.scratchDb.dropTable("SkyChip_xrad")
+       self.scratchDb.dropTable("SkyChip_xfit")
+       self.scratchDb.dropTable("SkyChip_xsrc")
+
        # create an output filename, which is {filterID}{skycellID}.FITS
-       self.outputFitsFile = "%08d.FITS" % self.stackID
+       self.outputFitsFile = "%08d.FITS" % self.id
        self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
 
@@ -67 +83 @@
 
        # insert what we know about this stack batch into the stack table
-       self.ippToPspsDb.insertStackMeta(self.batchID, self.id, self.stackID, self.filter, self.stackType)
+       self.ippToPspsDb.insertStackMeta(self.batchID, self.filter, self.stackType)
 
        # insert sourceID/imageID combo so DVO can look it up
@@ -80 +96 @@
        super(StackBatch, self).printMe()
 
-       self.logger.info("Sky ID:     %d" % self.id)
-       self.logger.info("Stack ID:   %d" % self.stackID)
-       self.logger.info("Stack type: %s" % self.stackType)
-       self.logger.info("Skycell:    %s" % self.skycell)
-       self.logger.info("Filter:     %s" % self.filter)
+       self.logger.infoPair("Stack ID", "%d" % self.id)
+       self.logger.infoPair("Stack type", "%s" % self.stackType)
+       self.logger.infoPair("Skycell", "%s" % self.skycell)
+       self.logger.infoPair("Filter", "%s" % self.filter)
 
 
@@ -92 +107 @@
     def updateStackMetaID(self, table):
 
-        sql = "UPDATE " + table + "  SET stackMetaID=" + str(self.stackID)
+        sql = "UPDATE " + table + "  SET stackMetaID=" + str(self.id)
         self.scratchDb.execute(sql)
 
@@ -225 +240 @@
     def populateStackMeta(self):
 
-        self.logger.info("Procesing StackMeta table")
+        self.logger.infoPair("Procesing table", "StackMeta")
 
         sql = "INSERT INTO StackMeta (\
@@ -247 +262 @@
         ,pc002002 \
          ) VALUES ( \
-        " + str(self.stackID) + " \
+        " + str(self.id) + " \
         ," + self.skycell + " \
         ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \
@@ -278 +293 @@
     def populateStackDetection(self):
 
-        self.logger.info("Procesing StackDetection table")
+        self.logger.infoPair("Procesing table", "StackDetection")
 
         # insert all the detections
@@ -390 +405 @@
     def populateStackApFlx(self):
 
-        self.logger.info("Procesing StackApFlx table")
+        self.logger.infoPair("Procesing", "StackApFlx")
  
         sql = "INSERT INTO StackApFlx \
@@ -403 +418 @@
 
         # TODO temporarily loading 1st convolved fluxes into unconvolved fields
-        self.logger.info("Adding un-convolved fluxes")
+        self.logger.infoPair("Adding fluxes", "un-convolved")
         self.updateApFlxs("", "< 7.0")
-        self.logger.info("Adding 1st convolved fluxes")
+        self.logger.infoPair("Adding fluxes", "1st convolved")
         self.updateApFlxs("c1", "< 7.0")
-        self.logger.info("Adding 2nd convolved fluxes")
+        self.logger.infoPair("Adding fluxes", "2nd convolved")
         self.updateApFlxs("c2", "> 7.0")
 
-        self.logger.info("Adding petrosians for extended sources")
+        self.logger.infoPair("Adding", "petrosians for extended sources")
         sql = "UPDATE StackApFlx AS a, SkyChip_xsrc AS b SET \
         petRadius=b.PETRO_RADIUS \
@@ -440 +455 @@
     def populateStackModelFit(self):
 
-        self.logger.info("Procesing StackModelFit table")
+        self.logger.infoPair("Procesing table", "StackModelFit")
 
         # insert all the detections
@@ -450 +465 @@
 
         # populate model parameters
-        self.logger.info("Adding deVaucouleurs fit")
+        self.logger.infoPair("Adding model fit", "deVaucouleurs")
         self.updateModelFit("deV", "PS_MODEL_DEV")
-        self.logger.info("Adding exponential fit")
+        self.logger.infoPair("Adding model fit", "exponential")
         self.updateModelFit("exp", "PS_MODEL_EXP")
-        self.logger.info("Adding sersic fit")
+        self.logger.infoPair("Adding model fit", "sersic")
         self.updateModelFit("ser", "PS_MODEL_SERSIC")
 
@@ -477 +492 @@
         rs.first()
         nMissing = rs.getInt(1)
-        self.logger.info("%5d detections in %s table that are not in StackDetection. Deleting" % (nMissing, table))
+        self.logger.infoPair("Detections deleted from %s that are not in StackDetection" + table, "%5d" % nMissing)
   
         if nMissing < 1: return
@@ -490 +505 @@
     def populateStackToImage(self):
 
-        self.logger.info("Procesing StackToImage table")
-
-        imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.stackID)
+        self.logger.infoPair("Procesing table", "StackToImage")
+
+        imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.id)
 
         for imageID in imageIDs:
             sql = "INSERT INTO StackToImage (stackMetaID, imageID) \
                    VALUES (\
-                   " + str(self.stackID) + ", " + imageID + ")"
+                   " + str(self.id) + ", " + imageID + ")"
             self.scratchDb.execute(sql)
 
@@ -509 +524 @@
     def populateSkinnyObject(self):
 
-        self.logger.info("Procesing SkinnyObject table")
+        self.logger.infoPair("Procesing table", "SkinnyObject")
 
         sql = "INSERT INTO SkinnyObject (\
@@ -529 +544 @@
     def populateObjectCalColor(self):
 
-        self.logger.info("Procesing ObjectCalColor table")
+        self.logger.infoPair("Procesing table", "ObjectCalColor table")
 
         sql = "INSERT INTO ObjectCalColor (\
@@ -550 +565 @@
     def alterPspsTables(self):
 
-        self.logger.info("Altering PSPS tables")
+        self.logger.debug("Altering PSPS tables")
         #self.scratchDb.makeColumnUnique("StackDetection", "objID")
         self.scratchDb.createIndex("StackDetection", "ippDetectID")
@@ -561 +576 @@
     def indexIppTables(self):
 
-        self.logger.info("Creating indexes on IPP tables")
+        self.logger.infoPair("Creating indexes on", "IPP tables")
         self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET")
         self.scratchDb.createIndex("SkyChip_xfit", "IPP_IDET")
@@ -575 +590 @@
 
         self.logger.debug("Updating table '" + table + "' with DVO IDs...")
-        sql = "UPDATE IGNORE " + table + " AS a, dvoDetectionFull AS b SET \
+        sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \
                a.ippObjID = b.ippObjID, \
                a.stackDetectID = b.detectID, \
@@ -613 +628 @@
         return True
 
-    '''
-    Checks whether this batch has already been processed and published
-    '''
-    def alreadyProcessed(self):
-
-        # sadly, we have to read the FITS primary header first
-        if not self.readPrimaryHeader(): return False
-
-        # get filterID using init table
-        self.filter = self.header['FPA.FILTER']
-        self.filter = self.filter[0:1]
-
-        self.stackType = stackType
-        meta = self.gpc1Db.getStackStageMeta(self.id, self.header['FPA.FILTER'])
-        if len(meta) < 1: return False
-        self.stackID = meta[0];
-        self.skycell = meta[1];
-        self.skycell = self.skycell[8:]
-        self.analysisVer = meta[2];
-
-        #return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID)
-        return False # TODOI
-
-
-logging.config.fileConfig("logging.conf")
-logger = logging.getLogger("stackbatch")
-logger.setLevel(logging.INFO)
-logger.info("Starting")
-
-gpc1Db = Gpc1Db(logger)
-ippToPspsDb = IppToPspsDb(logger)
-
-configDoc = ElementTree(file="config.xml")
-
-#stackType = "NIGHTLY_STACK"
-#skyIDs = gpc1Db.getIDsInThisDVODbForThisStageFudge()
-#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky")
-
-stackType = "DEEP_STACK"
-skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky")
-
-#skyIDs = [942]
-#skyIDs = [299]
-#skyIDs = [302]
-#skyIDs = [8508]
-#i = 0
-for skyID in skyIDs:
-    
-    #if skyID < 1340: continue # nightly
-    #if skyID < 238: continue # deep
-
-    cmfFiles = gpc1Db.getStackStageCmfs(skyID)
-
-    for file in cmfFiles:
-
-        stackBatch = StackBatch(logger,
-                                configDoc,
-                                gpc1Db,
-                                ippToPspsDb,
-                                skyID, 
-                                file, 
-                                stackType)
-
-        stackBatch.run()
-