Changeset 34735 for branches/eam_branches/ipp-20120905/ippToPsps/jython/stackbatch.py

Timestamp:

Nov 29, 2012, 2:15:01 PM (14 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20120905

Files:

• . (modified) (1 prop)
• ippToPsps (modified) (1 prop)
• ippToPsps/jython/stackbatch.py (modified) (19 diffs)

branches/eam_branches/ipp-20120905/ippToPsps/jython/stackbatch.py

@@ -76 +76 @@
        self.projectioncell = self.skycell
 
+       # proposed new values. Need to coordinate with the SkyCell table
+       # this fits in 32 bits for 0 < tessID < 214
+       # self.skycell = str( (tessID * (10000 * 1000)) + (int(self.projectioncell) * 1000) + cellID )
+
        self.analysisVer = meta[2];
 
@@ -89 +93 @@
        self.scratchDb.dropTable("SkinnyObject")
        self.scratchDb.dropTable("ObjectCalColor")
+       self.scratchDb.dropTable("StackDetectionCalib")
 
        # delete IPP tables
@@ -140 +145 @@
 
         sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET \
-        "+prefix+"flxR1=b.APER_FLUX_1, \
-        "+prefix+"flxR1Err=b.APER_FLUX_ERR_1, \
-        "+prefix+"flxR1Std=b.APER_FLUX_STDEV_1, \
+        "+prefix+"flxR1=b.APER_FLUX_1 / " + str(self.expTime) + ", \
+        "+prefix+"flxR1Err=b.APER_FLUX_ERR_1 / " + str(self.expTime) + ", \
+        "+prefix+"flxR1Std=b.APER_FLUX_STDEV_1 / " + str(self.expTime) + ", \
         "+prefix+"flxR1Fill=b.APER_FILL_1, \
-        "+prefix+"flxR2=b.APER_FLUX_2, \
-        "+prefix+"flxR2Err=b.APER_FLUX_ERR_2, \
-        "+prefix+"flxR2Std=b.APER_FLUX_STDEV_2, \
+        "+prefix+"flxR2=b.APER_FLUX_2 / " + str(self.expTime) + ", \
+        "+prefix+"flxR2Err=b.APER_FLUX_ERR_2 / " + str(self.expTime) + ", \
+        "+prefix+"flxR2Std=b.APER_FLUX_STDEV_2 / " + str(self.expTime) + ", \
         "+prefix+"flxR2Fill=b.APER_FILL_2, \
-        "+prefix+"flxR3=b.APER_FLUX_3, \
-        "+prefix+"flxR3Err=b.APER_FLUX_ERR_3, \
-        "+prefix+"flxR3Std=b.APER_FLUX_STDEV_3, \
+        "+prefix+"flxR3=b.APER_FLUX_3 / " + str(self.expTime) + ", \
+        "+prefix+"flxR3Err=b.APER_FLUX_ERR_3 / " + str(self.expTime) + ", \
+        "+prefix+"flxR3Std=b.APER_FLUX_STDEV_3 / " + str(self.expTime) + ", \
         "+prefix+"flxR3Fill=b.APER_FILL_3, \
-        "+prefix+"flxR4=b.APER_FLUX_4, \
-        "+prefix+"flxR4Err=b.APER_FLUX_ERR_4, \
-        "+prefix+"flxR4Std=b.APER_FLUX_STDEV_4, \
+        "+prefix+"flxR4=b.APER_FLUX_4 / " + str(self.expTime) + ", \
+        "+prefix+"flxR4Err=b.APER_FLUX_ERR_4 / " + str(self.expTime) + ", \
+        "+prefix+"flxR4Std=b.APER_FLUX_STDEV_4 / " + str(self.expTime) + ", \
         "+prefix+"flxR4Fill=b.APER_FILL_4, \
-        "+prefix+"flxR5=b.APER_FLUX_5, \
-        "+prefix+"flxR5Err=b.APER_FLUX_ERR_5, \
-        "+prefix+"flxR5Std=b.APER_FLUX_STDEV_5, \
+        "+prefix+"flxR5=b.APER_FLUX_5 / " + str(self.expTime) + ", \
+        "+prefix+"flxR5Err=b.APER_FLUX_ERR_5 / " + str(self.expTime) + ", \
+        "+prefix+"flxR5Std=b.APER_FLUX_STDEV_5 / " + str(self.expTime) + ", \
         "+prefix+"flxR5Fill=b.APER_FILL_5, \
-        "+prefix+"flxR6=b.APER_FLUX_6, \
-        "+prefix+"flxR6Err=b.APER_FLUX_ERR_6, \
-        "+prefix+"flxR6Std=b.APER_FLUX_STDEV_6, \
+        "+prefix+"flxR6=b.APER_FLUX_6 / " + str(self.expTime) + ", \
+        "+prefix+"flxR6Err=b.APER_FLUX_ERR_6 / " + str(self.expTime) + ", \
+        "+prefix+"flxR6Std=b.APER_FLUX_STDEV_6 / " + str(self.expTime) + ", \
         "+prefix+"flxR6Fill=b.APER_FILL_6, \
-        "+prefix+"flxR7=b.APER_FLUX_7, \
-        "+prefix+"flxR7Err=b.APER_FLUX_ERR_7, \
-        "+prefix+"flxR7Std=b.APER_FLUX_STDEV_7, \
+        "+prefix+"flxR7=b.APER_FLUX_7 / " + str(self.expTime) + ", \
+        "+prefix+"flxR7Err=b.APER_FLUX_ERR_7 / " + str(self.expTime) + ", \
+        "+prefix+"flxR7Std=b.APER_FLUX_STDEV_7 / " + str(self.expTime) + ", \
         "+prefix+"flxR7Fill=b.APER_FILL_7, \
-        "+prefix+"flxR8=b.APER_FLUX_8, \
-        "+prefix+"flxR8Err=b.APER_FLUX_ERR_8, \
-        "+prefix+"flxR8Std=b.APER_FLUX_STDEV_8, \
+        "+prefix+"flxR8=b.APER_FLUX_8 / " + str(self.expTime) + ", \
+        "+prefix+"flxR8Err=b.APER_FLUX_ERR_8 / " + str(self.expTime) + ", \
+        "+prefix+"flxR8Std=b.APER_FLUX_STDEV_8 / " + str(self.expTime) + ", \
         "+prefix+"flxR8Fill=b.APER_FILL_8, \
-        "+prefix+"flxR9=b.APER_FLUX_9, \
-        "+prefix+"flxR9Err=b.APER_FLUX_ERR_9, \
-        "+prefix+"flxR9Std=b.APER_FLUX_STDEV_9, \
+        "+prefix+"flxR9=b.APER_FLUX_9 / " + str(self.expTime) + ", \
+        "+prefix+"flxR9Err=b.APER_FLUX_ERR_9 / " + str(self.expTime) + ", \
+        "+prefix+"flxR9Std=b.APER_FLUX_STDEV_9 / " + str(self.expTime) + ", \
         "+prefix+"flxR9Fill=b.APER_FILL_9, \
-        "+prefix+"flxR10=b.APER_FLUX_10, \
-        "+prefix+"flxR10Err=b.APER_FLUX_ERR_10, \
-        "+prefix+"flxR10Std=b.APER_FLUX_STDEV_10, \
+        "+prefix+"flxR10=b.APER_FLUX_10 / " + str(self.expTime) + ", \
+        "+prefix+"flxR10Err=b.APER_FLUX_ERR_10 / " + str(self.expTime) + ", \
+        "+prefix+"flxR10Std=b.APER_FLUX_STDEV_10 / " + str(self.expTime) + ", \
         "+prefix+"flxR10Fill=b.APER_FILL_10 \
         WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition
@@ -196 +201 @@
         sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \
         "+model+"Radius=b.EXT_WIDTH_MAJ,  \
-        "+model+"Mag=b.EXT_INST_MAG,  \
-        "+model+"MagErr=b.EXT_INST_MAG_SIG,  \
+        "+model+"Flux=POW(10, -0.4 * b.EXT_INST_MAG) / " + str(self.expTime) + " ,  \
+        "+model+"FluxErr=ABS(b.EXT_INST_MAG_SIG) * POW(10, -0.4 * b.EXT_INST_MAG) / " + str(self.expTime) + " / 1.085736, \
         "+model+"Ab=b.EXT_WIDTH_MAJ/b.EXT_WIDTH_MIN, \
         "+model+"Phi=b.EXT_THETA,  \
@@ -319 +324 @@
                ,skyCellID \
                ,projectionCellID \
-               ,obsTime \
                ,xPos \
                ,yPos \
                ,xPosErr \
                ,yPosErr \
-               ,instFlux \
-               ,instFluxErr \
+               ,psfFlux \
+               ,psfFluxErr \
                ,peakFlux \
                ,sky \
@@ -335 +339 @@
                ,psfLikelihood \
                ,infoFlag \
-               ,psfCf \
+               ,psfQf \
+               ,psfQfPerfect \
                ,momentXX \
                ,momentXY \
@@ -345 +350 @@
                ,momentR1 \
                ,momentRH \
-               ,apMag \
-               ,apMagErr \
+               ,apFlux \
+               ,apFluxErr \
                ,kronFlux \
                ,kronFluxErr \
                ,kronRad \
-               ,kronRadErr \
+               ,psfChiSq \
                ,nFrames \
                ,assocDate \
@@ -359 +364 @@
                ," + self.skycell + " \
                ," + self.projectioncell + " \
-               ," + self.header['MJD-OBS'] + " \
                ,X_PSF \
                ,Y_PSF \
@@ -367 +371 @@
                ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + ")) / 1.085736) \
                ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + str(self.expTime) + " \
-               ,SKY \
-               ,SKY_SIGMA \
+               ,SKY / " + str(self.expTime) + " \
+               ,SKY_SIGMA / " + str(self.expTime) + " \
                ,EXT_NSIGMA \
                ,PSF_MAJOR \
@@ -374 +378 @@
                ,PSF_THETA \
                ,psfLikelihood(EXT_NSIGMA) \
-               ,FLAGS2 << 32 | FLAGS \
+               , FLAGS2 << 32 | FLAGS \
                ,PSF_QF \
+               ,PSF_QF_PERFECT \
                ,MOMENTS_XX \
                ,MOMENTS_XY \
@@ -385 +390 @@
                ,MOMENTS_R1 \
                ,MOMENTS_RH \
-               ,AP_MAG \
-               , NULL \
+               ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + " \
+               ,NULL \
                ,KRON_FLUX / " + str(self.expTime) + " \
                ,KRON_FLUX_ERR / " + str(self.expTime) + " \
-               , MOMENTS_R1 * 2.5 \
-               , NULL \
+               ,MOMENTS_R1 * 2.5 \
+               ,PSF_CHISQ \
                ,N_FRAMES \
                , '" + self.dateStr + "' \
@@ -417 +422 @@
         self.updateStackTypeID("StackDetection")
         
-        self.updateDvoIDs("StackDetection")
+        self.updateDvoIDsAndFlags("StackDetection")
         
         sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)"
@@ -427 +432 @@
             #if deep stack and instFlux = null and err not null
             sql = "UPDATE StackDetection AS a, SkyChip_psf AS b \
-                   SET instFlux = 2*b.PSF_INST_FLUX_SIG / " + str(self.expTime) + " \
-                   WHERE instFlux IS NULL \
+                   SET psfFlux = 2*b.PSF_INST_FLUX_SIG / " + str(self.expTime) + " \
+                   WHERE psfFlux IS NULL \
                    AND a.ippDetectID = b.IPP_IDET  \
                    AND b.PSF_INST_FLUX_SIG IS NOT NULL"
@@ -434 +439 @@
             self.scratchDb.execute(sql)
             
-        
-        self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux")
+        #leave null instflux in
+        #self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux")
         
         self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID")
@@ -474 +479 @@
         petRadius=b.PETRO_RADIUS \
         ,petRadiusErr=b.PETRO_RADIUS_ERR \
-        ,petMag=b.PETRO_MAG \
-        ,petMagErr=b.PETRO_MAG_ERR \
+        ,petFlux=POW(10.0, -0.4 * b.PETRO_MAG) /" + str(self.expTime) + " \
+        ,petFluxErr=ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.expTime) + " / 1.085736  \
         ,petR50=b.PETRO_RADIUS_50 \
         ,petR50Err=b.PETRO_RADIUS_50_ERR \
@@ -587 +592 @@
         self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.config.dataRelease))
 
+    '''    
+    Populates the StackDetectionCalib table
+    '''
+    def populateStackDetectionCalib(self):
+        self.logger.infoPair("Processing table", "StackDetectionCalib")
+        tableName = "StackDetectionCalib"
+        # insert all detections into table
+        sql = "INSERT INTO " + tableName + " ( \
+               objID \
+               ,stackDetectID \
+               ,ippObjID \
+               ,ippDetectID \
+               ,filterID \
+               ,surveyID \
+               ) \
+               SELECT \
+               objID \
+               ,stackDetectID \
+               ,ippObjID \
+               ,ippDetectID \
+               ,filterID \
+               ,surveyID \
+               FROM StackDetection"
+        self.scratchDb.execute(sql)
+        # insert calibration information from dvoDetections into the Table
+        sql = "UPDATE " + tableName + " AS a, "  + self.scratchDb.dvoDetectionTable + " AS b \
+            SET a.ra = b.ra, \
+            a.dec = b.dec_, \
+            a.raErr = b.raErr, \
+            a.decErr = b.decErr, \
+            a.zp = b.zp, \
+            a.zpErr = b.zpErr, \
+            a.expTime = b.expTime, \
+            a.airMass = b.airMass   \
+            WHERE a.stackDetectID = b.detectID"
+        self.scratchDb.execute(sql)
+        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
+        
+    
+
+
     '''
     Populates the ObjectCalColor table
@@ -620 +666 @@
         self.scratchDb.createIndex("StackModelFit", "ippDetectID")
 
+        # dec is reserved in MySQL, so STILTS replaces if with dec_, which is not the name of 
+        # the column in PSPS
+        self.scratchDb.execute("ALTER TABLE StackDetectionCalib CHANGE dec_ `dec` double")
+
         return True
 
@@ -645 +695 @@
         sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \
                a.ippObjID = b.ippObjID, \
-               a.stackDetectID = 10 * b.detectID + " + str(self.filterID) + ", \
+               a.stackDetectID = b.detectID, \
                a.objID = b.objID \
                WHERE a.ippDetectID = b.ippDetectID \
                AND b.imageID = " + str(imageID)
-        
-        self.scratchDb.execute(sql)
-        
+        self.scratchDb.execute(sql)
+    
+    '''
+    Updates Flags
+    '''
+    def updateDvoIDsAndFlags(self, table):
+        self.logger.infoPair("getting","imageID")
+        imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID'])
+        self.logger.infoPair("obtained","imageID")
+        self.logger.debug("Updating table '" + table + "' with DVO IDs...")
+        sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \
+               a.ippObjID = b.ippObjID, \
+               a.stackDetectID = b.detectID, \
+               a.objID = b.objID, \
+               a.infoFlag = b.flags << 45 | a.infoFlag \
+               WHERE a.ippDetectID = b.ippDetectID \
+               AND b.imageID = " + str(imageID)
+        self.scratchDb.execute(sql)
+    
+    
     '''
     Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
@@ -660 +727 @@
             if not self.getIDsFromDVO():
                 return False
-
+        #self.logger.infoPair("populating","StackMeta")    
         self.populateStackMeta()
+        #self.logger.infoPair("populating","StackDetection") 
         self.populateStackDetection()
-
+        
         if self.stackType != "NIGHTLY_STACK": 
+            #self.logger.infoPair("populating","StackModelFit") 
             self.populateStackModelFit()
+            #self.logger.infoPair("populating","StackApFlx") 
             self.populateStackApFlx()
-   
+        #self.logger.infoPair("populating","StackToImage") 
         self.populateStackToImage()
+        #self.logger.infoPair("populating","SkinnyObject") 
         self.populateSkinnyObject()
+        #self.logger.infoPair("populating","ObjectCalColor") 
         self.populateObjectCalColor()
+        #self.logger.infoPair("populating","StackDetectionCalib") 
+        self.populateStackDetectionCalib()
 
         self.setMinMaxObjID(["StackDetection"])