Changeset 31809 for trunk/ippToPsps/jython/batch.py

Timestamp:

Jul 1, 2011, 8:54:16 AM (15 years ago)

Author:

rhenders

Message:

implementing new dataRelease, useFullTables, force, testmode, publishToDatastore and reportNull options from config; now tarball creation done in same method as publish to datastore since; config XML document object passed into constructor; implemented printMe() method, which is overriden in all subclasses; improvements to logging messages

File:

• trunk/ippToPsps/jython/batch.py (modified) (12 diffs)

trunk/ippToPsps/jython/batch.py

@@ -34 +34 @@
     def __init__(self, 
                  logger, 
+                 doc,
                  gpc1Db,
                  ippToPspsDb,
-                 scratchDb,
                  id,
                  batchType, 
                  inputFitsPath="", 
-                 survey="", 
-                 useFullTables=False):
+                 survey=""): 
 
         self.everythingOK = False
         self.readHeader = False
+        self.doc = doc
 
         # set up logging
@@ -55 +55 @@
         self.gpc1Db = gpc1Db
         self.ippToPspsDb = ippToPspsDb
-        self.scratchDb = scratchDb
         self.batchType = batchType;
         self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot"
         self.inputFitsPath = inputFitsPath
         self.survey = survey
-        self.useFullTables = useFullTables
 
         if self.alreadyProcessed(): return
+
+        # get dvo info from config
+        dvoName = self.doc.find("dvo/name").text
+        self.dvoLocation = self.doc.find("dvo/location").text
+        self.useFullTables = int(self.doc.find("dvo/useFullTables").text)
+        self.scratchDb = ScratchDb(logger, self.useFullTables)
 
         # do we have an input file?
@@ -72 +76 @@
         self.tablesToExport = []
 
-        # open config
-        self.doc = ElementTree(file="config.xml")
-
         if self.survey != "":
             self.surveyID = self.scratchDb.getSurveyID(self.survey)
-   
-            # get dvo info from config
-            dvoName = self.doc.find("dvo_" + self.survey + "/name").text
-            self.dvoLocation = self.doc.find("dvo_" + self.survey + "/location").text
         else:
-            dvoName = ""
-            self.dvoLocation = ""
             self.surveyID = -1;
+
        
         # get some options from the config
-        self.dataRelease = self.doc.find("metadata/dataRelease").text
+        self.testMode = int(self.doc.find("options/testMode").text)
+        self.dataRelease = int(self.doc.find("metadata/dataRelease").text)
 
         # get datastore info from config
@@ -95 +92 @@
         self.batchID = self.ippToPspsDb.createNewBatch( 
                 self.getPspsBatchType(), 
+                self.id,
                 survey,
                 dvoName, 
@@ -120 +118 @@
 
         self.logger.debug("Batch destructor")
+
+    '''
+    Prints metadata to the log
+    '''
+    def printMe(self):
+
+        self.logger.info("New batch: ")
+        self.logger.info("Batch type:             " + self.batchType)
+        self.logger.info("Survey:                 " + self.survey)
+        self.logger.info("Survey ID:              %d" % self.surveyID)
+        self.logger.info("Test mode?              %d" % self.testMode)
+        self.logger.info("DVO location:           " + self.dvoLocation)
+        self.logger.info("Use full DVO tables?:   %d" % self.useFullTables)
+        self.logger.info("Input FITS file:        " + self.inputFitsPath)
 
 
@@ -240 +252 @@
 
     '''
-    tar and zips batch directory
-    '''
-    def createTarball(self):
+    Publishes this batch to the datastore
+    '''
+    def publishToDatastore(self):
       
-        if self.doc.find("options/createTarball").text == "false": return
-
         # set up filenams and paths
         tarFile = self.batchName + ".tar"
         tarPath = self.subDir + "/" + tarFile
 
-        self.tarballFile = tarFile + ".gz"
-        tarballPath = self.subDir + "/" + self.tarballFile
+        tarballFile = tarFile + ".gz"
+        tarballPath = self.subDir + "/" + tarballFile
 
         # tar directory
@@ -267 +277 @@
         shutil.rmtree(self.localOutPath)
 
-    '''
-    Publishes this batch to the datastore
-    '''
-    def publishToDatastore(self):
-
-        if self.doc.find("options/publishToDatastore").text == "false": return
-
-        if self.datastore.publish(self.batchName, self.subDir, self.tarballFile, "tgz"):
+        # now publish to the datastore
+        if self.datastore.publish(self.batchName, self.subDir, tarballFile, "tgz"):
             self.ippToPspsDb.updateLoadedToDatastore(self.batchID, 1)
 
@@ -282 +286 @@
     def getBatchFriendlySurveyType(self):
 
-        return "SCR" # TODO
-
+        #return "SCR" # TODO
         try:
             self.survey
@@ -395 +398 @@
     def importIppTables(self, filter=""):
 
-      self.logger.info("Attempting to import tables from input FITS file")
+      self.logger.info("Attempting to import tables from input FITS file with regex " + filter)
       tables = stilts.treads(self.inputFitsPath)
 
@@ -433 +436 @@
         _tables = []
 
-        self.logger.info("Selecting database tables")
+        self.logger.info("Selecting database tables for export")
         for table in self.tablesToExport:
 
@@ -472 +475 @@
     def reportNullsInAllPspsTables(self, showPartials):
 
-        if self.doc.find("options/reportNulls").text == "false": return
-
         for table in self.pspsTables:
             self.scratchDb.reportNulls(table.name, showPartials)
@@ -527 +528 @@
 
         self.createEmptyPspsTables()
-        #self.importIppTables()
+        self.importIppTables()
         if self.populatePspsTables():
             if self.exportPspsTablesToFits():
                 self.writeBatchManifest()
-                self.createTarball()
-                self.publishToDatastore()
-                self.reportNullsInAllPspsTables(False)
-                #sys.exit()
+                if int(self.doc.find("options/publishToDatastore").text): self.publishToDatastore()
+                if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False)
+
         self.logger.info("Finished.")
-
-
+        if self.testMode: sys.exit()
+
+