Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py (revision 37159) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py (revision 37160) @@ -242,4 +242,16 @@ if key in header: return header[key] + else: + self.logger.errorPair("Missing header field", key) + return "NULL" + + ''' + Returns the string keyword value from this header or else "NULL" + ''' + def getKeyFloat(self, header, format, key): + + if key in header: + value = format % float(header[key]) + return value else: self.logger.errorPair("Missing header field", key) Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/config.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/config.py (revision 37159) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/config.py (revision 37160) @@ -24,4 +24,6 @@ self.resetDvo = False self.gpc1test = True + self.retry = False + self.camera = "gpc1" if sys.argv.count("-test"): @@ -47,7 +49,17 @@ sys.argv.remove("-reset-dvo") + if sys.argv.count("-retry"): + self.retry = True + sys.argv.remove("-retry") + + if sys.argv.count("-simtest"): + self.camera = "simtest" + sys.argv.remove("-simtest") + print "test: " + str(self.test) print "gpc1test: " + str(self.gpc1test) print "resetDvo: " + str(self.resetDvo) + print "retry: " + str(self.retry) + print "camera: " + self.camera for arg in sys.argv: Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/dvo.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/dvo.py (revision 37159) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/dvo.py (revision 37160) @@ -679,8 +679,8 @@ useP2 = 1 - if (useP2 ==0 and useST ==1): + if (useP2 == 0 and useST == 1): # grab only stacks - if self.config.gpc1test: - # gpc1test -> SIMTEST.x.SkyChip + if (self.config.camera == "simtest"): + # SIMTEST.x.SkyChip cmd += " -photcode-start 14100 -photcode-end 14500" else: @@ -688,8 +688,8 @@ cmd += " -photcode-start 11000 -photcode-end 11500" - if (useP2 ==1 and useST==0): - #grab only P2s (4000-4500 are test cameras, SIMTEST and SIMMOSAIC) - if self.config.gpc1test: - # gpc1test -> SIMTEST.x.Chip + if (useP2 == 1 and useST == 0): + # grab only P2s (4000-4500 are test cameras, SIMTEST and SIMMOSAIC) + if (self.config.camera == "simtest"): + # SIMTEST.x.Chip cmd += " -photcode-start 4100 -photcode-end 4500" else: Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/mysql.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/mysql.py (revision 37159) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/mysql.py (revision 37160) @@ -283,5 +283,5 @@ #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'") - def addPrimaryKeyWithIgnore(self, table, column) + def addPrimaryKeyWithIgnore(self, table, column): # note that in mysql versions later than 5.1, this fails # unless the following is called first: Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py (revision 37159) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py (revision 37160) @@ -103,5 +103,5 @@ if not (str(stackID) == header['IMAGEID']): - self.logger.errorPair("inconsistency: header IMAGEID does not match stackID", stackID + " vs " + header['IMAGEID']) + self.logger.errorPair("inconsistency: header IMAGEID does not match stackID", str(stackID) + " vs " + header['IMAGEID']) raise @@ -173,14 +173,15 @@ # delete IPP tables - self.logger.info("dropping Stack cmf tables") - for filter in self.filters: - tableName = filter + "SkyChip_psf" - self.scratchDb.dropTable(tableName) - tableName = filter + "SkyChip_xrad" - self.scratchDb.dropTable(tableName) - tableName = filter + "SkyChip_xsrc" - self.scratchDb.dropTable(tableName) - tableName = filter + "SkyChip_xfit" - self.scratchDb.dropTable(tableName) + if not self.config.retry: + self.logger.info("dropping Stack cmf tables") + for filter in self.filters: + tableName = filter + "SkyChip_psf" + self.scratchDb.dropTable(tableName) + tableName = filter + "SkyChip_xrad" + self.scratchDb.dropTable(tableName) + tableName = filter + "SkyChip_xsrc" + self.scratchDb.dropTable(tableName) + tableName = filter + "SkyChip_xfit" + self.scratchDb.dropTable(tableName) # create an output filename, which is {stackID}.FITS @@ -350,10 +351,10 @@ self.logger.infoPair("Populating table", tablename) - self.fwhm_maj = self.getKeyValue(header, 'FWHM_MAJ') - self.fwhm_maj_uq = self.getKeyValue(header, 'FW_MJ_UQ') - self.psfmodel = self.getKeyValue(header, 'PSFMODEL') - - if (self.fwhm_maj == "NULL"): self.fwhm_maj = -999 - if (self.fwhm_maj_uq == "NULL"): self.fwhm_maj_uq = -999 + fwhm_maj = self.getKeyValue(header, 'FWHM_MAJ') + fwhm_maj_uq = self.getKeyValue(header, 'FW_MJ_UQ') + psfmodel = self.getKeyValue(header, 'PSFMODEL') + + if (fwhm_maj == "NULL"): fwhm_maj = -999 + if (fwhm_maj_uq == "NULL"): fwhm_maj_uq = -999 # make a table @@ -363,9 +364,7 @@ photoCalID = str(self.scratchDb.getPhotoCalID(stackID)) - # XXX mysql is sensitive to values which are ambiugously float. eg + # mysql is sensitive to values which are ambiugously float. eg # a warning is raised if we try to insert '25.' into a float field. - # I should make a function to sanitize float values to a desired number - # of decimal points - zpString = "%5.5f" % float(header['FPA.ZP']) + # use getKeyFloat(hdr, format, key) to avoid this problem sql = "CREATE TABLE " + tablename + " LIKE StackMeta" @@ -382,21 +381,21 @@ sqlLine.group("skyCellID", self.skycell) sqlLine.group("photoCalID", photoCalID) - sqlLine.group("photoZero", zpString) # XXX see note above - sqlLine.group("expTime", header['EXPTIME']) - sqlLine.group("psfModelID", self.psfmodel) - sqlLine.group("psfFwhm_mean", str(self.fwhm_maj)) - sqlLine.group("psfFwhm_max", str(self.fwhm_maj_uq)) + sqlLine.group("photoZero", self.getKeyFloat(header, "%.5f", 'FPA.ZP')) + sqlLine.group("expTime", self.getKeyFloat(header, "%.5f", 'EXPTIME')) + sqlLine.group("psfModelID", psfmodel) + sqlLine.group("psfFwhm_mean", fwhm_maj) + sqlLine.group("psfFwhm_max", fwhm_maj_uq) sqlLine.group("ctype1", header['CTYPE1']) sqlLine.group("ctype2", header['CTYPE2']) - sqlLine.group("crval1", header['CRVAL1']) - sqlLine.group("crval2", header['CRVAL2']) - sqlLine.group("crpix1", header['CRPIX1']) - sqlLine.group("crpix2", header['CRPIX2']) - sqlLine.group("cdelt1", header['CDELT1']) - sqlLine.group("cdelt2", header['CDELT2']) - sqlLine.group("pc001001", header['PC001001']) - sqlLine.group("pc001002", header['PC001002']) - sqlLine.group("pc002001", header['PC002001']) - sqlLine.group("pc002002", header['PC002002']) + sqlLine.group("crval1", self.getKeyFloat(header, "%.8f", 'CRVAL1')) + sqlLine.group("crval2", self.getKeyFloat(header, "%.8f", 'CRVAL2')) + sqlLine.group("crpix1", self.getKeyFloat(header, "%.8f", 'CRPIX1')) + sqlLine.group("crpix2", self.getKeyFloat(header, "%.8f", 'CRPIX2')) + sqlLine.group("cdelt1", self.getKeyFloat(header, "%.8e", 'CDELT1')) + sqlLine.group("cdelt2", self.getKeyFloat(header, "%.8e", 'CDELT2')) + sqlLine.group("pc001001", self.getKeyFloat(header, "%.8e", 'PC001001')) + sqlLine.group("pc001002", self.getKeyFloat(header, "%.8e", 'PC001002')) + sqlLine.group("pc002001", self.getKeyFloat(header, "%.8e", 'PC002001')) + sqlLine.group("pc002002", self.getKeyFloat(header, "%.8e", 'PC002002')) sql = sqlLine.make(") VALUES ( ", ")") @@ -469,5 +468,5 @@ # insert values coming from the CMF, not carried by DVO header = self.headerSet[filter] - exptime = header['EXPTIME'] + exptime = self.getKeyFloat(header, "%.5f", "EXPTIME") sqlLine = sqlUtility("UPDATE StackObjectThin AS a , " + filter + "SkyChip_psf AS b SET") @@ -516,5 +515,5 @@ header = self.headerSet[filter] - exptime = header['EXPTIME'] + exptime = self.getKeyFloat(header, "%.5f", "EXPTIME") # insert all the detections @@ -603,15 +602,21 @@ for filter in self.filters: - populateStackModelFitFilter(model, ippModelType, filter) + self.populateStackModelFitFilter(model, ippModelType, filter) ''' Updates parameters for a particular model in the StackModelFit table ''' - def populateStackModelFitFilter(self, model, ippModelType, filter, exptime): + def populateStackModelFitFilter(self, model, ippModelType, filter): + + if self.stackIDs[filter] <= 0: + self.logger.infoPair("no stack data for filter" , filter) + return True + + table = "StackModelFit" + model header = self.headerSet[filter] - exptime = header['EXPTIME'] - - sqlLine = sqlUtility("UPDATE StackModelFit AS a, " + filter + "SkyChip_xfit AS b SET") + exptime = self.getKeyFloat(header, "%.5f", "EXPTIME") + + sqlLine = sqlUtility("UPDATE " + table + " AS a, " + filter + "SkyChip_xfit AS b SET") # these lines yield the flux, flux error values, but I do not think we want those in the database... @@ -630,9 +635,9 @@ sqlLine.group("a." + filter + model + "PhiErr", "b.EXT_THETA_ERR") # these are not correctly defined. - sqlLine.group("a." + filter + model + "Ra", "b.RA_EXT") - sqlLine.group("a." + filter + model + "Dec", "b.DEC_EXT") - - sqlLine.group("a." + filter + model + "RaErr", "b.X_EXT_SIG * b.PLTSCALE") - sqlLine.group("a." + filter + model + "DecErr", "b.Y_EXT_SIG * b.PLTSCALE") + # XXX this is tricky: RA_EXT is not defined, but X_EXT is... + # sqlLine.group("a." + filter + model + "Ra", "b.RA_EXT") + # sqlLine.group("a." + filter + model + "Dec", "b.DEC_EXT") + # sqlLine.group("a." + filter + model + "RaErr", "b.X_EXT_SIG * b.PLTSCALE") + # sqlLine.group("a." + filter + model + "DecErr", "b.Y_EXT_SIG * b.PLTSCALE") # sersic fit has an extra parameter @@ -874,5 +879,5 @@ self.logger.infoPair("Creating indexes on", "IPP tables") - self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") self.scratchDb.createIndex("SkyChip_xfit", "IPP_IDET") self.scratchDb.createIndex("SkyChip_xrad", "IPP_IDET") @@ -1091,54 +1096,56 @@ ''' def importIppTables(self, columns="*", tableRE=""): - - self.logger.infoPair("Importing ST tables with table match expression: ", tableRE) - - count = 0 - for filter in self.filters: - if self.stackIDs[filter] <= 0: - self.logger.infoPair("no files for filter" , filter) - continue - - fileName = self.fits[filter].getPath() - - self.logger.infoPair("using filename:",fileName) - - try: - tables = stilts.treads(fileName) - except: - self.logger.errorPair("STILTS could not import from", fileName) - return False + + if self.config.retry: return True + + self.logger.infoPair("Importing ST tables with table match expression: ", tableRE) + + count = 0 + for filter in self.filters: + if self.stackIDs[filter] <= 0: + self.logger.infoPair("no files for filter" , filter) + continue + + fileName = self.fits[filter].getPath() + + self.logger.infoPair("using filename:",fileName) + + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + + # count = 0 + for table in tables: - # count = 0 - for table in tables: - - # print "import smf table ", table - match = re.match(tableRE, table.name) - if not match: continue - - self.logger.infoPair("Reading IPP table", filter + table.name) - table = stilts.tpipe(table, cmd='addcol table_index $0') - table = stilts.tpipe(table, cmd='explodeall') + # print "import smf table ", table + match = re.match(tableRE, table.name) + if not match: continue + + self.logger.infoPair("Reading IPP table", filter + table.name) + table = stilts.tpipe(table, cmd='addcol table_index $0') + table = stilts.tpipe(table, cmd='explodeall') - # print "read smf table ", table + # print "read smf table ", table + + # drop any previous tables before import + self.scratchDb.dropTable(filter + table.name) - # drop any previous tables before import - self.scratchDb.dropTable(filter + table.name) + # IPP FITS files are littered with infinities, so remove these + self.logger.debug("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + # print "cleaned up values ", table - # IPP FITS files are littered with infinities, so remove these - self.logger.debug("Removing Infinity values from all columns") - table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') - table = stilts.tpipe(table, cmd='replaceval -Infinity null *') - table = stilts.tpipe(table, cmd='replaceval Infinity null *') - # print "cleaned up values ", table - - try: - table.write(self.scratchDb.url + '#' + filter + table.name) - count = count + 1 - except: - self.logger.exception("Problem writing table '" + filter + table.name + "' to the database") + try: + table.write(self.scratchDb.url + '#' + filter + table.name) + count = count + 1 + except: + self.logger.exception("Problem writing table '" + filter + table.name + "' to the database") - self.logger.infoPair("Done. Imported", "%d tables" % count) - self.indexIppTables() + self.logger.infoPair("Done. Imported", "%d tables" % count) + self.indexIppTables() - return True + return True