Changeset 34041 for branches/meh_branches/ppstack_test/ippToPsps/jython/batch.py

Timestamp:

Jun 19, 2012, 5:24:19 PM (14 years ago)

Author:

mhuber

Message:

merging latest r34040 trunk changes to branch

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• ippToPsps/jython/batch.py (modified) (10 diffs)

branches/meh_branches/ppstack_test/ippToPsps/jython/batch.py

@@ -10 +10 @@
 import logging
 from subprocess import call, PIPE, Popen
+from java.lang import *
+from java.sql import *
+from xml.etree.ElementTree import ElementTree, Element, tostring
 
 from scratchdb import ScratchDb
@@ -15 +18 @@
 from ipptopspsdb import IppToPspsDb
 from fits import Fits
-
-from java.lang import *
-from java.sql import *
-from xml.etree.ElementTree import ElementTree, Element, tostring
 
 '''
@@ -40 +39 @@
                  useFullTables): 
 
-        self.everythingOK = False
         self.readHeader = False
         self.config = config
@@ -67 +65 @@
             if not self.header: 
                 logger.errorPair("Could not read FITS for id", "%d" % id)
-                return
-
-        # if no fits file, and this is not an IN batch, then get out of here
-        else:
-           if batchType != "IN": 
-               logger.errorPair("Could not read FITS for id", "%d" % id)
-               return
+                raise
 
         self.scratchDb.setUseFullTables(self.useFullTables)
@@ -122 +114 @@
         self.logger.infoPair("Output path", self.localOutPath)
 
-        self.everythingOK = True
     
     '''
@@ -241 +232 @@
         root.attrib['type'] = self.batchType
         root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") 
-        if self.batchType != "IN": root.attrib['survey'] = self.config.pspsSurvey
+        root.attrib['survey'] = self.config.pspsSurvey
+
+        # min/max object IDs
         try: self.minObjID
         except: pass
@@ -260 +253 @@
 
         # clunky way to prettify XML
-        p = Popen("xmllint --format " + tmpPath + " > " + outPath, shell=True, stdout=PIPE)
-        p.wait()
-        os.remove(tmpPath)
+        try:
+            cmd = "xmllint --format " + tmpPath + " > " + outPath
+            p = Popen(cmd, shell=True, stdout=PIPE)
+            p.wait()
+        except:
+            self.logger.errorPair("Popen failed", cmd)
+            return False
+        finally: 
+            os.remove(tmpPath)
+
+        return True
 
 
@@ -296 +297 @@
             return False
 
-        # only now can we report that the batch has successfully processed
-        self.ippToPspsDb.updateProcessed(self.batchID, 1)
-
         # delete tar file and original directory
         os.remove(tarPath)
@@ -410 +408 @@
       self.logger.infoPair("Done. Imported", "%d tables" % count)
       self.indexIppTables()
+
+      return True
 
     '''
@@ -508 +508 @@
     def run(self):
 
-        if not self.everythingOK:
-            self.logger.error("Aborting this batch due to (hopefully) previously reported errors")
+        if not self.createEmptyPspsTables():
+            self.logger.errorPair("Aborting this batch", "could not create empty PSPS tables")
             self.ippToPspsDb.updateProcessed(self.batchID, -1)
-            return
-
-        if not self.createEmptyPspsTables():
-            self.logger.error("Aborting this batch due to (hopefully) previously reported errors")
+            raise
+
+        if not self.importIppTables():
+            self.logger.errorPair("Aborting this batch", "could not import IPP tables")
             self.ippToPspsDb.updateProcessed(self.batchID, -1)
-            return
-
-        self.importIppTables()
+            raise
+
         if not self.populatePspsTables():
             self.logger.errorPair("Aborting this batch", "unable to populate PSPS tables")
             self.ippToPspsDb.updateProcessed(self.batchID, -1)
-        else:
-            if not self.exportPspsTablesToFits():
-                self.logger.errorPair("Aborting this batch", "unable to export tables to FITS file")
-            else:
-                self.writeBatchManifest()
-                if self.config.datastorePublishing: 
-                    # tar and zip ready for publication to datastore
-                    if self.tarAndZip():
-                        # now publish to the datastore
-                        tarballFile = Batch.getTarballFile(self.batchID)
-                        Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile)
+            raise 
+
+        if not self.exportPspsTablesToFits():
+            self.logger.errorPair("Aborting this batch", "unable to export tables to FITS file")
+            self.ippToPspsDb.updateProcessed(self.batchID, -1)
+            raise 
+   
+        if self.writeBatchManifest():
+            if self.config.datastorePublishing: 
+
+                # tar and zip ready for publication to datastore
+                if self.tarAndZip():
+
+                    # only now can we report that the batch has successfully processed
+                    self.ippToPspsDb.updateProcessed(self.batchID, 1)
+                    
+                    # now publish to the datastore
+                    tarballFile = Batch.getTarballFile(self.batchID)
+                    Batch.publishToDatastore(
+                            self.datastore, 
+                            self.batchID, 
+                            self.batchName, 
+                            self.subDir, 
+                            tarballFile)
 
 from datastore import Datastore