Index: trunk/ippToPsps/jython/batch.py =================================================================== --- trunk/ippToPsps/jython/batch.py (revision 31809) +++ trunk/ippToPsps/jython/batch.py (revision 31840) @@ -15,4 +15,5 @@ from gpc1db import Gpc1Db from ipptopspsdb import IppToPspsDb +from fits import Fits from java.lang import * @@ -39,5 +40,5 @@ id, batchType, - inputFitsPath="", + fits, survey=""): @@ -45,8 +46,10 @@ self.readHeader = False self.doc = doc + self.fits = fits + self.header = self.fits.getPrimaryHeader() # set up logging self.logger = logger - self.logger.info("-------------------------------------------------------------------------------") + self.logger.infoSeparator() self.logger.debug("Batch class constructor") @@ -57,8 +60,5 @@ self.batchType = batchType; self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot" - self.inputFitsPath = inputFitsPath self.survey = survey - - if self.alreadyProcessed(): return # get dvo info from config @@ -66,10 +66,5 @@ self.dvoLocation = self.doc.find("dvo/location").text self.useFullTables = int(self.doc.find("dvo/useFullTables").text) - self.scratchDb = ScratchDb(logger, self.useFullTables) - - # do we have an input file? - if self.inputFitsPath != "": - - if not self.readPrimaryHeader(): return + self.scratchDb = ScratchDb(logger, self.doc, self.useFullTables) # TODO @@ -124,39 +119,13 @@ def printMe(self): - self.logger.info("New batch: ") - self.logger.info("Batch type: " + self.batchType) - self.logger.info("Survey: " + self.survey) - self.logger.info("Survey ID: %d" % self.surveyID) - self.logger.info("Test mode? %d" % self.testMode) - self.logger.info("DVO location: " + self.dvoLocation) - self.logger.info("Use full DVO tables?: %d" % self.useFullTables) - self.logger.info("Input FITS file: " + self.inputFitsPath) - - - ''' - Reads the primary header of the FITS file - ''' - def readPrimaryHeader(self): - - if self.readHeader: return True - - # does it exist? - if not os.path.isfile(self.inputFitsPath): - - self.logger.error("Cannot read file at '" + self.inputFitsPath + "'") - return False - - file = open(self.inputFitsPath) - self.header = self.parseFitsHeader(file) - self.logger.info("Read primary header and found " + str(len(self.header)) + " header cards") - # TODO close file? - - self.readHeader = True - - return True - - - ''' - Returns the value from this dictinary or else NULL + self.logger.infoPair("Batch name", self.batchName) + self.logger.infoPair("Survey", self.survey) + self.logger.infoPair("Survey ID", "%d" % self.surveyID) + self.logger.infoPair("DVO location", self.dvoLocation) + self.logger.infoPair("Use full DVO tables?", "%d" % self.useFullTables) + self.logger.infoPair("Output path", self.localOutPath) + + ''' + Returns the string value from this dictionary or else "NULL" ''' def safeDictionaryAccess(self, header, key): @@ -166,40 +135,4 @@ ''' - Finds and reads a header extension - ''' - def findAndReadFITSHeader(self, name, file): - - found = False - - origIndex = file.tell() - while True: - - index = file.tell() - - record = file.read(80) - if not record: break; - - # quit when we reach 'END' - if record.startswith("XTENSION= 'IMAGE"): - - header = self.parseFitsHeader(file) - if header['EXTNAME'] == name: - found = True - file.seek(index + 2880, 0) - break - - file.seek(index + 2880, 0) - - if found != True: - - file.seek(origIndex, 0) - self.logger.error("...could not read header in extension '" + name + "'") - return - #else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards") - - return header - - - ''' Writes the batch manifest file ''' @@ -208,5 +141,5 @@ outPath = self.localOutPath + "/BatchManifest.xml" tmpPath = "./tmp.xml" - self.logger.info("Creating batch manifest file here: " + outPath) + self.logger.infoPair("Creating manifest", outPath) root = Element('manifest') @@ -299,5 +232,5 @@ ''' - Gets PSPS friendly batch type + Gets PSPS friendly batch type TODO only use PSPS batch names, P2, ST etc ''' def getPspsBatchType(self): @@ -335,33 +268,7 @@ self.ippToPspsDb.updateMinMaxObjID(self.batchID, self.minObjID, self.maxObjID) - self.logger.info("Total detections = %ld min objID = %ld max objID = %ld" % (self.totalDetections, self.minObjID, self.maxObjID)) - - - ''' - Reads FITS header and stores all fields in a dictionary object - ''' - def parseFitsHeader(self, fitsFile): - - header = {} - - while (True): - - record = fitsFile.read(80) - - # quit when we reach 'END' - if re.match('END\s+', record): break - - # this regex will get param/value pairs for all header cards, ignoring comments and parsing out 'HIERACH' prefixes - match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-_\.:\s@#]+)\'*\\/*', record) - if match: - - param = match.group(2) - value = match.group(3).strip() - if value == "NaN": value = "NULL" - header[param] = value - - #print param + "|" + value + "|" - - return header + self.logger.infoPair("Total detections", "%ld" % self.totalDetections) + self.logger.infoPair("Min objID", "%ld" % self.minObjID) + self.logger.infoPair("Max objID", "%ld" % self.maxObjID) ''' @@ -398,6 +305,6 @@ def importIppTables(self, filter=""): - self.logger.info("Attempting to import tables from input FITS file with regex " + filter) - tables = stilts.treads(self.inputFitsPath) + self.logger.infoPair("Importing tables with filter", filter) + tables = stilts.treads(self.fits.getPath()) count = 0 @@ -406,5 +313,5 @@ match = re.match(filter, table.name) if not match: continue - self.logger.info("Reading IPP table " + table.name + " from FITS file") + self.logger.infoPair("Reading IPP table", table.name) table = stilts.tpipe(table, cmd='explodeall') @@ -425,5 +332,5 @@ - self.logger.info("Done. Imported %d tables" % count) + self.logger.infoPair("Done. Imported", "%d tables" % count) self.indexIppTables() @@ -433,8 +340,9 @@ def exportPspsTablesToFits(self, regex="(.*)"): - self.logger.info("Replacing NULLs with -999, changing tables names using regex: " + regex) + self.logger.infoPair("Replacing NULLs with", "-999") + self.logger.infoPair("Changing table names with regex", regex) _tables = [] - self.logger.info("Selecting database tables for export") + self.logger.debug("Selecting database tables for export") for table in self.tablesToExport: @@ -460,5 +368,5 @@ _tables.append(_table) - self.logger.info("Writing to FITS file '" + self.outputFitsPath + "'...") + self.logger.infoPair("Writing to FITS", self.outputFitsPath) try: stilts.twrites(_tables, self.outputFitsPath, fmt='fits') @@ -483,8 +391,8 @@ def replaceAllPspsNulls(self, sub): - self.logger.info("Replacing all NULL values in PSPS tables with '" + sub + "'...") + self.logger.infoPair("Replacing all NULL values with", sub) for table in self.pspsTables: self.scratchDb.replaceNulls(table.name, sub) - self.logger.info("...done") + self.logger.infoPair("NULL replacement", "done") ''' @@ -501,9 +409,9 @@ # TODO path to DVO prog hardcoded temporarily cmd = "../src/dvograbber " + self.dvoLocation - self.logger.info("Running: '" + cmd + "'...") + self.logger.infoPair("Running DVO", cmd) p = Popen(cmd, shell=True, stdout=PIPE) p.wait() # out = p.stdout.read() - self.logger.info("...done") + self.logger.infoPair("DVO access", "complete") if self.scratchDb.getRowCount("dvoDetection") < 1: @@ -517,5 +425,5 @@ ''' def alreadyProcessed(self): - self.logger.info("Not implemented") + self.logger.error("Not implemented") @@ -535,6 +443,6 @@ if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False) - self.logger.info("Finished.") - if self.testMode: sys.exit() - - + self.logger.infoPair("Batch......", "complete") + self.logger.infoSeparator() + +