Changeset 31286 for trunk/ippToPsps/jython/batch.py

Timestamp:

Apr 14, 2011, 3:17:49 PM (15 years ago)

Author:

rhenders

Message:

removing Infinity values prior to entry into database; method to call DVO C program to get IDs; improved logic for constructor args; only writing certain fields to manifest when defined

File:

• trunk/ippToPsps/jython/batch.py (modified) (8 diffs)

trunk/ippToPsps/jython/batch.py

@@ -60 +60 @@
         self.gpc1Db = Gpc1Db(self.logger)
 
+        if self.survey != "":
+
         # get survey ID from init table
-        sql = "SELECT surveyID from Survey WHERE name = '" + survey + "'"
-        try:
-            rs = self.localStmt.executeQuery(sql)  
-            rs.first()
-            self.surveyID = rs.getInt(1)
-        except:
-            self.logger.exception("No survey ID found for this survey: '" + survey + "'")
-            self.surveyID = -1;
+            sql = "SELECT surveyID from Survey WHERE name = '" + self.getPspsSurveyType() + "'"
+            try:
+                rs = self.localStmt.executeQuery(sql)  
+                rs.first()
+                self.surveyID = rs.getInt(1)
+            except:
+                self.logger.exception("No survey ID found for this survey: '" + self.getPspsSurveyType() + "'")
+                self.surveyID = -1;
    
         # get dvo info from config
-        if survey != "":
             dvoName = doc.find(survey+"dvo/name").text
-            dvoLocation = doc.find(survey+"dvo/location").text
+            self.dvoLocation = doc.find(survey+"dvo/location").text
         else:
             dvoName = ""
-            dvoLocation = ""
+            self.dvoLocation = ""
+            self.surveyID = -1;
          
         # get datastore info from config
@@ -127 +129 @@
         root.attrib['name'] = self.batchName
         root.attrib['type'] = self.getPspsBatchType()
-        root.attrib['survey'] = self.getPspsSurveyType()
         root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") 
-        root.attrib['minObjId'] = str(self.minObjID)
-        root.attrib['maxObjId'] = str(self.maxObjID)
+        if self.survey != "":
+            root.attrib['survey'] = self.getPspsSurveyType()
+        try: self.minObjID
+        except: pass
+        else: root.attrib['minObjId'] = str(self.minObjID)
+        try: self.maxObjID
+        except: pass
+        else: root.attrib['maxObjId'] = str(self.maxObjID)
 
         # get md5sum
@@ -198 +205 @@
     def getPspsSurveyType(self):
 
+        try:
+            self.survey
+        except:
+            return "NA" 
+
         if self.survey == "ThreePi": return "3PI"
-        elif self.survey == "MD04": return "MD04"
-        else: self.logger.error("Don't know this survey: " + self.survey)
+        elif self.survey == "MD04": return "MD4"
+        else:
+            self.logger.error("Don't know this survey: '" + self.survey + "'")
+            return "NA"
 
     '''
@@ -232 +246 @@
         self.maxObjID = rs.getLong(2)
 
-        print "MIN mAX = %d %d" % (self.minObjID,  self.maxObjID)
-
     '''
     Updates a table with surveyID
@@ -332 +344 @@
     Imports IPP tables from FITS file
 
-    Accepts a regular expression filter so not all tabls need to be imported
+    Accepts a regular expression filter so not all tables need to be imported
     '''
     def importIppTables(self, filter):
 
       tables = stilts.treads(self.inputFitsPath)
+      self.logger.info("Attempting to import tables from input FITS file")
 
       count = 0
@@ -343 +356 @@
           match = re.match(filter, table.name)
           if not match: continue
-          self.logger.info("Creating IPP table " + table.name)
+          self.logger.info("   Reading IPP table " + table.name + " from FITS file")
           table = stilts.tpipe(table, cmd='explodeall')
+
+          # IPP FITS files are littered with infinities, so remove these
+          self.logger.info("   Removing Infinity values from all columns")
+          table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
+          table = stilts.tpipe(table, cmd='replaceval Infinity null *')
+
           try:
               table.write(self.localUrl + '#' + table.name)
           except:
-              self.logger.exception("Problem writing table '" + table.name + "' to the database")
+              self.logger.exception("   Problem writing table '" + table.name + "' to the database")
 
           count = count + 1
 
-      self.logger.info("Imported %d tables from '%s' " % (count, self.inputFitsPath))
+      self.logger.info("Done. Imported %d tables" % count)
 
       self.indexIppTables()
@@ -367 +386 @@
         for table in self.pspsTables:
            _table = stilts.tread(self.localUrl + '#SELECT * FROM ' + table.name)
+           #           if table.name == "StackDetection": _table.cmd_keepcols('stackDetectID projectionCellID').cmd_head(4).write()
+
+           self.logger.info("   Replacing NULLs with weird PSPS -999 constant for " + table.name)
+           _table = stilts.tpipe(_table, cmd='replaceval "" -999 *') # TODO doesn't work
+           #if table.name == "StackDetection": _table.cmd_keepcols('stackDetectID projectionCellID').cmd_head(4).write()
            _table = stilts.tpipe(_table, cmd='tablename ' + table.name)
            _tables.append(_table)
@@ -480 +504 @@
         sql = "DROP TABLE dvo"
         self.localStmt.execute(sql)
-        sql = "CREATE TABLE dvo (ippDetectID BIGINT PRIMARY KEY, ippObjID BIGINT, objID BIGINT)"
+        sql = "CREATE TABLE dvo (ippDetectID BIGINT PRIMARY KEY, detectID BIGINT, ippObjID BIGINT, objID BIGINT)"
         self.localStmt.execute(sql)
 
+
+    '''
+    Calls DVO program to 'query' DVO database and populate results to local MySQL Db table
+    '''
+    def getIDsFromDVO(self, sourceID, imageID):
+
+        # TODO path to DVO prog hardcoded temporarily
+        cmd = "../src/dvo %s %s %s" % (self.dvoLocation, sourceID, imageID)
+        self.logger.info("Running: '" + cmd + "'...")
+        p = Popen(cmd, shell=True, stdout=PIPE)
+        p.wait()
+        #        out = p.stdout.read()
+        self.logger.info("...done")
+