Changeset 31840 for trunk/ippToPsps/jython/batch.py

Timestamp:

Jul 8, 2011, 2:48:02 PM (15 years ago)

Author:

rhenders

Message:

removed FITS specific code and moved into new Fits class; config XML document now passed-in to all database constructors; using new logging class, PSLogger

File:

• trunk/ippToPsps/jython/batch.py (modified) (19 diffs)

trunk/ippToPsps/jython/batch.py

@@ -15 +15 @@
 from gpc1db import Gpc1Db
 from ipptopspsdb import IppToPspsDb
+from fits import Fits
 
 from java.lang import *
@@ -39 +40 @@
                  id,
                  batchType, 
-                 inputFitsPath="", 
+                 fits, 
                  survey=""): 
 
@@ -45 +46 @@
         self.readHeader = False
         self.doc = doc
+        self.fits = fits
+        self.header = self.fits.getPrimaryHeader()
 
         # set up logging
         self.logger = logger
-        self.logger.info("-------------------------------------------------------------------------------")
+        self.logger.infoSeparator()
         self.logger.debug("Batch class constructor")
 
@@ -57 +60 @@
         self.batchType = batchType;
         self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot"
-        self.inputFitsPath = inputFitsPath
         self.survey = survey
-
-        if self.alreadyProcessed(): return
 
         # get dvo info from config
@@ -66 +66 @@
         self.dvoLocation = self.doc.find("dvo/location").text
         self.useFullTables = int(self.doc.find("dvo/useFullTables").text)
-        self.scratchDb = ScratchDb(logger, self.useFullTables)
-
-        # do we have an input file?
-        if self.inputFitsPath != "":
-
-            if not self.readPrimaryHeader(): return
+        self.scratchDb = ScratchDb(logger, self.doc, self.useFullTables)
 
         # TODO
@@ -124 +119 @@
     def printMe(self):
 
-        self.logger.info("New batch: ")
-        self.logger.info("Batch type:             " + self.batchType)
-        self.logger.info("Survey:                 " + self.survey)
-        self.logger.info("Survey ID:              %d" % self.surveyID)
-        self.logger.info("Test mode?              %d" % self.testMode)
-        self.logger.info("DVO location:           " + self.dvoLocation)
-        self.logger.info("Use full DVO tables?:   %d" % self.useFullTables)
-        self.logger.info("Input FITS file:        " + self.inputFitsPath)
-
-
-    '''
-    Reads the primary header of the FITS file
-    '''
-    def readPrimaryHeader(self):
-
-        if self.readHeader: return True
-
-        # does it exist?
-        if not os.path.isfile(self.inputFitsPath):
-
-            self.logger.error("Cannot read file at '" + self.inputFitsPath + "'")
-            return False
-
-        file = open(self.inputFitsPath)
-        self.header = self.parseFitsHeader(file)
-        self.logger.info("Read primary header and found " + str(len(self.header)) + " header cards")
-        # TODO close file?
-
-        self.readHeader = True
-
-        return True
-
-
-    '''
-    Returns the value from this dictinary or else NULL
+        self.logger.infoPair("Batch name", self.batchName)
+        self.logger.infoPair("Survey", self.survey)
+        self.logger.infoPair("Survey ID", "%d" % self.surveyID)
+        self.logger.infoPair("DVO location", self.dvoLocation)
+        self.logger.infoPair("Use full DVO tables?", "%d" % self.useFullTables)
+        self.logger.infoPair("Output path", self.localOutPath)
+
+    '''
+    Returns the string value from this dictionary or else "NULL"
     '''
     def safeDictionaryAccess(self, header, key):
@@ -166 +135 @@
 
     '''
-    Finds and reads a header extension
-    '''
-    def findAndReadFITSHeader(self, name, file):
-
-        found = False
-       
-        origIndex = file.tell()
-        while True:
-           
-            index = file.tell()
-
-            record = file.read(80)
-            if not record: break;
-
-            # quit when we reach 'END'
-            if record.startswith("XTENSION= 'IMAGE"):
-
-                header = self.parseFitsHeader(file)
-                if header['EXTNAME'] == name:
-                    found = True
-                    file.seek(index + 2880, 0)
-                    break
-
-            file.seek(index + 2880, 0)
-            
-        if found != True:
-
-            file.seek(origIndex, 0)
-            self.logger.error("...could not read header in extension '" + name + "'")
-            return
-        #else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards") 
-
-        return header
-
-
-    '''
     Writes the batch manifest file
     '''
@@ -208 +141 @@
         outPath = self.localOutPath + "/BatchManifest.xml"
         tmpPath = "./tmp.xml"
-        self.logger.info("Creating batch manifest file here: " + outPath)
+        self.logger.infoPair("Creating manifest", outPath)
         root = Element('manifest')
 
@@ -299 +232 @@
 
     '''
-    Gets PSPS friendly batch type
+    Gets PSPS friendly batch type TODO only use PSPS batch names, P2, ST etc
     '''
     def getPspsBatchType(self):
@@ -335 +268 @@
 
         self.ippToPspsDb.updateMinMaxObjID(self.batchID, self.minObjID, self.maxObjID)
-        self.logger.info("Total detections = %ld min objID = %ld max objID = %ld" % (self.totalDetections, self.minObjID, self.maxObjID))
-
-
-    '''
-    Reads FITS header and stores all fields in a dictionary object
-    '''
-    def parseFitsHeader(self, fitsFile):
-
-        header = {}
-
-        while (True):
-
-           record = fitsFile.read(80)
-
-           # quit when we reach 'END'
-           if re.match('END\s+', record): break
-
-           # this regex will get param/value pairs for all header cards, ignoring comments and parsing out 'HIERACH' prefixes
-           match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-_\.:\s@#]+)\'*\\/*', record)
-           if match:
-
-               param = match.group(2)
-               value = match.group(3).strip()
-               if value == "NaN": value = "NULL"
-               header[param] = value
-
-               #print param + "|" + value + "|"
-
-        return header
+        self.logger.infoPair("Total detections", "%ld" % self.totalDetections)
+        self.logger.infoPair("Min objID", "%ld" % self.minObjID)
+        self.logger.infoPair("Max objID", "%ld" % self.maxObjID)
 
     '''
@@ -398 +305 @@
     def importIppTables(self, filter=""):
 
-      self.logger.info("Attempting to import tables from input FITS file with regex " + filter)
-      tables = stilts.treads(self.inputFitsPath)
+      self.logger.infoPair("Importing tables with filter", filter)
+      tables = stilts.treads(self.fits.getPath())
 
       count = 0
@@ -406 +313 @@
           match = re.match(filter, table.name)
           if not match: continue
-          self.logger.info("Reading IPP table " + table.name + " from FITS file")
+          self.logger.infoPair("Reading IPP table", table.name)
           table = stilts.tpipe(table, cmd='explodeall')
 
@@ -425 +332 @@
 
 
-      self.logger.info("Done. Imported %d tables" % count)
+      self.logger.infoPair("Done. Imported", "%d tables" % count)
       self.indexIppTables()
 
@@ -433 +340 @@
     def exportPspsTablesToFits(self, regex="(.*)"):
 
-        self.logger.info("Replacing NULLs with -999, changing tables names using regex: " + regex)
+        self.logger.infoPair("Replacing NULLs with", "-999")
+        self.logger.infoPair("Changing table names with regex", regex)
         _tables = []
 
-        self.logger.info("Selecting database tables for export")
+        self.logger.debug("Selecting database tables for export")
         for table in self.tablesToExport:
 
@@ -460 +368 @@
            _tables.append(_table)
 
-        self.logger.info("Writing to FITS file '" + self.outputFitsPath + "'...")
+        self.logger.infoPair("Writing to FITS", self.outputFitsPath)
         try:
             stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
@@ -483 +391 @@
     def replaceAllPspsNulls(self, sub):
 
-        self.logger.info("Replacing all NULL values in PSPS tables with '" + sub + "'...")
+        self.logger.infoPair("Replacing all NULL values with", sub)
         for table in self.pspsTables:
             self.scratchDb.replaceNulls(table.name, sub)
-        self.logger.info("...done")
+        self.logger.infoPair("NULL replacement", "done")
 
     '''
@@ -501 +409 @@
         # TODO path to DVO prog hardcoded temporarily
         cmd = "../src/dvograbber " + self.dvoLocation
-        self.logger.info("Running: '" + cmd + "'...")
+        self.logger.infoPair("Running DVO", cmd)
         p = Popen(cmd, shell=True, stdout=PIPE)
         p.wait()
         #        out = p.stdout.read()
-        self.logger.info("...done")
+        self.logger.infoPair("DVO access", "complete")
 
         if self.scratchDb.getRowCount("dvoDetection") < 1:
@@ -517 +425 @@
     '''
     def alreadyProcessed(self):
-        self.logger.info("Not implemented")
+        self.logger.error("Not implemented")
 
 
@@ -535 +443 @@
                 if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False)
 
-        self.logger.info("Finished.")
-        if self.testMode: sys.exit()
-
-
+        self.logger.infoPair("Batch......", "complete")
+        self.logger.infoSeparator()
+
+