Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py (revision 37137) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py (revision 37138) @@ -381,5 +381,5 @@ self.tablesToExport.append(table.name) - return self.alterPspsTables(); + return True ''' @@ -391,5 +391,5 @@ ''' - Alter PSPS tables + Alter PSPS tables (Subclass implements this) ''' def alterPspsTables(self): @@ -447,94 +447,4 @@ self.indexIppTables() - return True - ''' - Imports IPP tables from FITS file - - Accepts a regular expression filter so not all tables need to be imported - ''' - def importIppSTTables(self, columns="*", filter=""): - - self.logger.infoPair("Importing ST tables with filter", filter) - - # print "trying to read ", self.fits.getPath() - - # ST has 5 cmf files - all with the same table names. To counter this we do it like this. - - filters = ["g","r","i","z","y"] - count = 0 - for f in filters: - skip = 0 - if (f == "g"): - if self.gstackID > 0: - fileName = self.gfits.getPath() - else: - skip = 1 - if (f == "r"): - if self.rstackID > 0: - fileName = self.rfits.getPath() - else: - skip = 1 - if (f == "i"): - if self.istackID > 0: - fileName = self.ifits.getPath() - else: - skip = 1 - if (f == "z"): - if self.zstackID > 0: - fileName = self.zfits.getPath() - else: - skip = 1 - if (f == "y"): - if self.ystackID > 0: - fileName = self.yfits.getPath() - else: - skip = 1 - - if skip == 1: - self.logger.infoPair("no files for filter" , f) - else: - self.logger.infoPair("using filename:",fileName) - - try: - tables = stilts.treads(fileName) - except: - self.logger.errorPair("STILTS could not import from", fileName) - return False - - #count = 0 - for table in tables: - - # print "import smf table ", table - match = re.match(filter, table.name) - - if not match: continue - self.logger.infoPair("Reading IPP table", f + table.name) - table = stilts.tpipe(table, cmd='addcol table_index $0') - - table = stilts.tpipe(table, cmd='explodeall') - - # print "read smf table ", table - - # drop any previous tables before import - self.scratchDb.dropTable(f + table.name) - - # IPP FITS files are littered with infinities, so remove these - self.logger.debug("Removing Infinity values from all columns") - table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') - table = stilts.tpipe(table, cmd='replaceval -Infinity null *') - table = stilts.tpipe(table, cmd='replaceval Infinity null *') - # print "cleaned up values ", table - - try: - table.write(self.scratchDb.url + '#' + f + table.name) - # XXX this one is not needed - # self.scratchDb.killLastConnectionID() - count = count + 1 - except: - self.logger.exception("Problem writing table '" + f + table.name + "' to the database") - - self.logger.infoPair("Done. Imported", "%d tables" % count) - self.indexIppTables() - return True @@ -641,6 +551,5 @@ def run(self): - # this is badly named : it creates the tables and then fills in - # the ImageMeta tables for each ota (alterPspsTables) + # creates the empty PSPS tables if not self.createEmptyPspsTables(): self.logger.errorPair("Aborting this batch", "could not create empty PSPS tables") @@ -648,23 +557,21 @@ raise + # fill in basic, not cmf/smf data (eg, ImageMeta for detectionbatch) + if not self.alterPspsTables(): + self.logger.errorPair("Aborting this batch", "could not alter PSPS tables for batch type") + self.ippToPspsDb.updateProcessed(self.batchID, -1) + raise + # for P2/ST, this reads the detection tables from the CMF/SMF file(s) # for OB, this imports object data from DVO - if self.batchType == "ST": - self.logger.infoPair("did we get here","?") - if not self.importIppSTTables(): - self.logger.errorPair("skipping this batch", "could not import ST IPP tables") - self.ippToPspsDb.updateProcessed(self.batchID, -1) - raise - self.logger.infoPair("we did the ST","yes") - else: - # if not ST we do this - if not self.importIppTables(): - if self.batchType == "OB": - self.logger.errorPair("skipping this batch", "no dvo tables for this region") - self.ippToPspsDb.updateProcessed(self.batchID, 1) - return True - self.logger.errorPair("Aborting this batch", "could not import IPP tables") - self.ippToPspsDb.updateProcessed(self.batchID, -1) - raise + if not self.importIppTables(): + if self.batchType == "OB": + self.logger.errorPair("skipping this batch", "no dvo tables for this region") + self.ippToPspsDb.updateProcessed(self.batchID, 1) + return True + self.logger.errorPair("Aborting this batch", "could not import IPP tables") + self.ippToPspsDb.updateProcessed(self.batchID, -1) + raise + self.logger.infoPair("populatePspsTables","ok") if not self.populatePspsTables(): @@ -700,4 +607,5 @@ tarballFile) +# XXX why is this down here?? from datastore import Datastore Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py (revision 37137) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py (revision 37138) @@ -75,5 +75,5 @@ # self.logger.infoPair("y stackID ",ystackID) # raise - # from each stackget fits name from getStackStageCmf + # from each stack, get fits name from getStackStageCmf if gstackID > 0: self.logger.infoPair ("finding fits file for: ","g.00000") @@ -141,14 +141,10 @@ metastackid = -999 - if gstackID > 0: - metastackid = gstackID - if rstackID > 0: - metastackid = rstackID - if istackID > 0: - metastackid = istackID - if zstackID > 0: - metastackid = zstackID - if ystackID > 0: - metastackid = ystackID + if gstackID > 0: metastackid = gstackID + if rstackID > 0: metastackid = rstackID + if istackID > 0: metastackid = istackID + if zstackID > 0: metastackid = zstackID + if ystackID > 0: metastackid = ystackID + #there is probably a better way to do this if metastackid < 0: @@ -159,5 +155,4 @@ ## this should be ok - it takes a stack_id and returns filter/skycell etc. - if not meta: self.logger.errorPair("Could not get stack", "metadata") @@ -174,4 +169,5 @@ # we need 1133.081 for skycell, but store as an int, so instead store 1133081 # projection cell is just the 1133 bit + # This Is how it should be done #self.projectioncell = self.skycell[8:12] @@ -217,8 +213,7 @@ self.scratchDb.dropTable("StackToImage") - # self.testprint() + # self.testprint() # for PV2 we will have (grizy) of stack meta and skychip - # delete IPP tables @@ -1335,2 +1330,91 @@ def exportPspsTablesToFits(self, regex="(.*)"): return super(StackBatch, self).exportPspsTablesToFits("(Stack.*)") + + ''' + Imports IPP tables from FITS file + Accepts a regular expression to match the tables so not all tables need to be imported + (This regex feature is not currently used...) + ''' + def importIppTables(self, columns="*", tableRE=""): + + self.logger.infoPair("Importing ST tables with table match expression: ", tableRE) + + # print "trying to read ", self.fits.getPath() + + # ST has 5 cmf files - all with the same table names. To counter this we do it like this. + + filters = ["g","r","i","z","y"] + count = 0 + for f in filters: + skip = 0 + if (f == "g"): + if self.gstackID > 0: + fileName = self.gfits.getPath() + else: + skip = 1 + if (f == "r"): + if self.rstackID > 0: + fileName = self.rfits.getPath() + else: + skip = 1 + if (f == "i"): + if self.istackID > 0: + fileName = self.ifits.getPath() + else: + skip = 1 + if (f == "z"): + if self.zstackID > 0: + fileName = self.zfits.getPath() + else: + skip = 1 + if (f == "y"): + if self.ystackID > 0: + fileName = self.yfits.getPath() + else: + skip = 1 + + if skip == 1: + self.logger.infoPair("no files for filter" , f) + else: + self.logger.infoPair("using filename:",fileName) + + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + + #count = 0 + for table in tables: + + # print "import smf table ", table + match = re.match(tableRE, table.name) + + if not match: continue + self.logger.infoPair("Reading IPP table", f + table.name) + table = stilts.tpipe(table, cmd='addcol table_index $0') + + table = stilts.tpipe(table, cmd='explodeall') + + # print "read smf table ", table + + # drop any previous tables before import + self.scratchDb.dropTable(f + table.name) + + # IPP FITS files are littered with infinities, so remove these + self.logger.debug("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + # print "cleaned up values ", table + + try: + table.write(self.scratchDb.url + '#' + f + table.name) + count = count + 1 + except: + self.logger.exception("Problem writing table '" + f + table.name + "' to the database") + + self.logger.infoPair("Done. Imported", "%d tables" % count) + self.indexIppTables() + + return True