IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 31469


Ignore:
Timestamp:
May 6, 2011, 12:42:21 PM (15 years ago)
Author:
rhenders
Message:

Hacks to kill last MySQL connection opened by STILTS functions; new run() method; now counting total detections as well as min/max objID; changed some constructor logic

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/batch.py

    r31405 r31469  
    3232    "../config/2/tables.vot"
    3333    '''
    34     def __init__(self, logger, batchType, inputFitsPath="", survey="", useFullTables=False):
     34    def __init__(self,
     35                 logger,
     36                 gpc1Db,
     37                 ippToPspsDb,
     38                 scratchDb,
     39                 id,
     40                 batchType,
     41                 inputFitsPath="",
     42                 survey="",
     43                 useFullTables=False):
     44
     45        self.everythingOK = False
     46        self.readHeader = False
    3547
    3648        # set up logging
    … …  
    4052
    4153        # set up class variables
     54        self.id = id
     55        self.gpc1Db = gpc1Db
     56        self.ippToPspsDb = ippToPspsDb
     57        self.scratchDb = scratchDb
    4258        self.batchType = batchType;
    4359        self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot"
    … …  
    4662        self.useFullTables = useFullTables
    4763
     64        if self.alreadyProcessed(): return
     65
     66        # do we have an input file?
     67        if self.inputFitsPath != "":
     68
     69            if not self.readPrimaryHeader(): return
     70
    4871        # TODO
    4972        self.tablesToExport = []
    … …  
    5174        # open config
    5275        doc = ElementTree(file="config.xml")
    53 
    54         # create Gpc1Db object
    55         self.gpc1Db = Gpc1Db(self.logger)
    56         self.ippToPspsDb = IppToPspsDb(logger)
    57         self.scratchDb = ScratchDb(logger, self.useFullTables)
    5876
    5977        if self.survey != "":
    … …  
    88106        self.dateStr = now.strftime("%Y-%m-%d")
    89107
    90         if self.inputFitsPath != "":
    91             file = open(self.inputFitsPath)
    92             self.header = self.parseFitsHeader(file)
    93             self.logger.info("Read primary and found " + str(len(self.header)) + " header cards")
    94             # TODO close file?
    95 
    96108        # create DVO tables if accessing DVO directly
    97109        if not self.useFullTables: self.scratchDb.createDvoTables()
    98110
     111        self.everythingOK = True
     112
    99113    '''
    100114    Destructor
    … …  
    103117
    104118        self.logger.debug("Batch destructor")
     119
     120
     121    '''
     122    Reads the primary header of the FITS file
     123    '''
     124    def readPrimaryHeader(self):
     125
     126        if self.readHeader: return True
     127
     128        # does it exist?
     129        if not os.path.isfile(self.inputFitsPath):
     130
     131            self.logger.error("Cannot read file at '" + self.inputFitsPath + "'")
     132            return False
     133
     134        file = open(self.inputFitsPath)
     135        self.header = self.parseFitsHeader(file)
     136        self.logger.info("Read primary header and found " + str(len(self.header)) + " header cards")
     137        # TODO close file?
     138
     139        self.readHeader = True
     140
     141        return True
    105142
    106143
    … …  
    138175            file.seek(index + 2880, 0)
    139176           
    140         if found != True: self.logger.error("...could not find extension '" + name + "'")
    141         else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards")
     177        if found != True:
     178            self.logger.error("...could not read header in extension '" + name + "'")
     179            return
     180        #else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards")
    142181
    143182        return header
    … …  
    262301
    263302        first = True
     303
     304        self.totalDetections = 0
    264305        for table in tables:
    265306
    266             sql = "SELECT MIN(objID), MAX(objID) FROM " + table
    267             rs = self.scratchDb.stmt.executeQuery(sql)
     307            sql = "SELECT MIN(objID), MAX(objID), COUNT(objID) FROM " + table
     308            rs = self.scratchDb.executeQuery(sql)
    268309            rs.first()
     310
     311            self.totalDetections = self.totalDetections + rs.getLong(3)
    269312
    270313            if first:
    … …  
    276319
    277320            first = False
     321            rs.close()
    278322
    279323        self.ippToPspsDb.updateMinMaxObjID(self.batchID, self.minObjID, self.maxObjID)
     324        self.logger.info("Total detections = %ld min objID = %ld max objID = %ld" % (self.totalDetections, self.minObjID, self.maxObjID))
     325
    280326
    281327    '''
    … …  
    313359         self.pspsTables = stilts.treads(self.pspsVoTableFilePath)
    314360         for table in self.pspsTables:
    315              self.logger.info("Creating PSPS table: " + table.name)
     361             self.logger.debug("Creating PSPS table: " + table.name)
    316362             table.write(self.scratchDb.url + '#' + table.name)
    317363             self.tablesToExport.append(table.name)
    … …  
    337383    Accepts a regular expression filter so not all tables need to be imported
    338384    '''
    339     def importIppTables(self, filter):
     385    def importIppTables(self, filter=""):
    340386
    341387      self.logger.info("Attempting to import tables from input FITS file")
    … …  
    347393          match = re.match(filter, table.name)
    348394          if not match: continue
    349           self.logger.info("   Reading IPP table " + table.name + " from FITS file")
     395          self.logger.info("Reading IPP table " + table.name + " from FITS file")
    350396          table = stilts.tpipe(table, cmd='explodeall')
    351397
    352398          # drop any previous tables before import
    353           self.scratchDb.dropTable(table.name)
     399          #self.scratchDb.dropTable(table.name)
    354400
    355401          # IPP FITS files are littered with infinities, so remove these
    356           self.logger.info("   Removing Infinity values from all columns")
     402          self.logger.debug("Removing Infinity values from all columns")
    357403          table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
    358404          table = stilts.tpipe(table, cmd='replaceval Infinity null *')
    … …  
    360406          try:
    361407              table.write(self.scratchDb.url + '#' + table.name)
     408              self.scratchDb.killLastConnectionID()
     409              count = count + 1
    362410          except:
    363               self.logger.exception("   Problem writing table '" + table.name + "' to the database")
    364           count = count + 1
     411              self.logger.exception("Problem writing table '" + table.name + "' to the database")
     412
    365413
    366414      self.logger.info("Done. Imported %d tables" % count)
    367 
    368415      self.indexIppTables()
    369416
    … …  
    373420    def exportPspsTablesToFits(self, regex="(.*)"):
    374421
    375         self.logger.info("Replacing NULLs with -999 then exporting all PSPS tables to FITS")
     422        self.logger.info("Replacing NULLs with -999, changing tables names using regex: " + regex)
    376423        _tables = []
    377424
    378         self.logger.info("    Selecting database tables")
     425        self.logger.info("Selecting database tables")
    379426        for table in self.tablesToExport:
    380427
    … …  
    383430
    384431           # get everything from table
    385            _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
    386 
     432           try:
     433               _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
     434               self.scratchDb.killLastConnectionID()
     435           except:
     436               self.logger.exception("Could not read from DB table: " + table)
     437               return False
     438               
    387439           # replace nulls and empty fields with weird PSPS -999 pseudo-null
    388440           _table = stilts.tpipe(_table, cmd='replaceval "" -999 *')
    … …  
    395447           _tables.append(_table)
    396448
    397         self.logger.info("    Writing to FITS file '" + self.outputFitsPath + "'...")
    398         stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
    399         self.logger.info("    ...done")
    400         self.ippToPspsDb.updateProcessed(self.batchID, 1)
     449        self.logger.info("Writing to FITS file '" + self.outputFitsPath + "'...")
     450        try:
     451            stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
     452            self.ippToPspsDb.updateProcessed(self.batchID, 1)
     453        except:
     454            self.logger.exception("Could not write to FITS")
     455            return False
     456
     457        return True
    401458
    402459    '''
    … …  
    447504    '''
    448505    def alreadyProcessed(self):
    449            self.logger.info("Not implemented")
    450 
    451 
    452 
     506        self.logger.info("Not implemented")
     507
     508
     509    '''
     510    Creates and publishes a batch
     511    '''
     512    def run(self):
     513
     514        if not self.everythingOK: return
     515
     516        self.createEmptyPspsTables()
     517        self.importIppTables()
     518        if self.populatePspsTables():
     519            if self.exportPspsTablesToFits():
     520                self.writeBatchManifest()
     521                self.createTarball()
     522                self.publishToDatastore()
     523                #self.reportNullsInAllPspsTables(False)
     524                #sys.exit()
     525        self.logger.info("Finished.")
     526
     527
Note: See TracChangeset for help on using the changeset viewer.