Changeset 31222

Timestamp:

Apr 6, 2011, 10:25:10 PM (15 years ago)

Author:

rhenders

Message:

Added logging; methods to create tarball and publish to datastore

File:

• trunk/ippToPsps/jython/batch.py (modified) (24 diffs)

trunk/ippToPsps/jython/batch.py

@@ -7 +7 @@
 import os
 import md5
-
+import shutil
+import logging
+from subprocess import call, PIPE, Popen
+
+from datastore import Datastore
 from gpc1db import Gpc1Db
 from ipptopspsdb import IppToPspsDb
@@ -14 +18 @@
 from java.sql import *
 from xml.etree.ElementTree import ElementTree, Element, tostring
-from xml.dom import minidom
 
 '''
@@ -30 +33 @@
     "../config/2/tables.vot"
     '''
-    def __init__(self, batchType, inputFitsPath="", survey=""):
+    def __init__(self, logger, batchType, inputFitsPath="", survey=""):
+
+        # set up logging
+        self.logger = logger
+        self.logger.debug("Batch class constructor")
 
         # set up class variables
@@ -51 +58 @@
 
         # create Gpc1Db object
-        self.gpc1Db = Gpc1Db()
+        self.gpc1Db = Gpc1Db(self.logger)
 
         # get survey ID from init table
@@ -60 +67 @@
             self.surveyID = rs.getInt(1)
         except:
-            self.log("No survey ID found for this survey: '" + survey + "'")
+            self.logger.exception("No survey ID found for this survey: '" + survey + "'")
             self.surveyID = -1;
    
         # get dvo info from config
-        dvoName = doc.find(survey+"dvo/name").text
-        dvoLocation = doc.find(survey+"dvo/location").text
+        if survey != "":
+            dvoName = doc.find(survey+"dvo/name").text
+            dvoLocation = doc.find(survey+"dvo/location").text
+        else:
+            dvoName = ""
+            dvoLocation = ""
          
         # get datastore info from config
-        datastoreProduct = doc.find("datastore/product").text
+        self.datastore = Datastore(self.logger)
 
         # create IppToPspsDb object and create a new batch
-        self.ippToPspsDb = IppToPspsDb()
-        self.batchID = self.ippToPspsDb.createNewBatch(66, survey, self.getPspsBatchType(), dvoName, datastoreProduct)
+        self.ippToPspsDb = IppToPspsDb(logger)
+        self.batchID = self.ippToPspsDb.createNewBatch(66, 
+                survey, 
+                self.getPspsBatchType(), 
+                dvoName, 
+                self.datastore.product)
 
         # get local storage location from config
         self.batchName = "B%08d" % self.batchID
-        self.localOutPath = doc.find("localOutPath").text + "/" + self.getPspsBatchType() + "/" + dvoName + "/%s" % self.batchName 
+        self.subDir = doc.find("localOutPath").text + "/" + self.getPspsBatchType() + "/" + dvoName
+        self.localOutPath = self.subDir + "/" + self.batchName 
         if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath)
 
@@ -94 +110 @@
     def __del__(self):
 
-        self.log("Batch destructor")
+        self.logger.debug("Batch destructor")
         self.localStmt.close()
         self.localCon.close()
@@ -105 +121 @@
         outPath = self.localOutPath + "/BatchManifest.xml"
         tmpPath = "./tmp.xml"
-        self.log("Creating batch manifest file here: " + outPath)
+        self.logger.info("Creating batch manifest file here: " + outPath)
         root = Element('manifest')
 
@@ -117 +133 @@
 
         # get md5sum
-        fits=os.popen("md5sum " + self.outputFitsPath)
-        for f in fits.readlines(): md5sum = f[0:f.rfind(" ")]
+        p = Popen("md5sum " + self.outputFitsPath, shell=True, stdout=PIPE)
+        p.wait()
+        out = p.stdout.read()
+        md5sum = out[0:out.rfind(" ")]
 
         # get file size
@@ -136 +154 @@
 
         # clunky way to prettify XML
-        os.popen("xmllint --format " + tmpPath + " > " + outPath)
-#        os.remove(tmpPath)
-
+        p = Popen("xmllint --format " + tmpPath + " > " + outPath, shell=True, stdout=PIPE)
+        p.wait()
+        os.remove(tmpPath)
+
+
+    '''
+    tar and zips batch directory
+    '''
+    def createTarball(self):
+      
+        # set up filenams and paths
+        tarFile = self.batchName + ".tar"
+        tarPath = self.subDir + "/" + tarFile
+
+        self.tarballFile = tarFile + ".gz"
+        tarballPath = self.subDir + "/" + self.tarballFile
+
+        # tar directory
+        p = Popen("tar -cvf " + tarPath + "\
+                -C " + self.subDir + " \
+                " + self.batchName, shell=True, stdout=PIPE)
+        p.wait()
+
+        # zip tar archive
+        p = Popen("gzip -c " + tarPath + " > " + tarballPath, shell=True, stdout=PIPE)
+        p.wait()
+
+        # delete tar file and original directory
+        os.remove(tarPath)
+        shutil.rmtree(self.localOutPath)
+
+    '''
+    Publishes this batch to the datastore
+    '''
+    def publishToDatastore(self):
+
+        self.datastore.publish(self.batchName, self.subDir, self.tarballFile, "tgz")
+        # TODO update ippToPsps Db here
 
     '''
@@ -147 +200 @@
         if self.survey == "ThreePi": return "3PI"
         elif self.survey == "MD04": return "MD04"
-        else: self.log("ERROR: Don't know this survey: " + self.survey)
+        else: self.logger.error("Don't know this survey: " + self.survey)
 
     '''
@@ -157 +210 @@
         elif self.batchType == "detection": return "P2"
         elif self.batchType == "stack": return "ST"
-        else: self.log("ERROR: Don't know this batch type: " + self.survey)
+        else: self.logger.error("Don't know this batch type: " + self.survey)
 
 
@@ -244 +297 @@
          self.pspsTables = stilts.treads(self.pspsVoTableFilePath)
          for table in self.pspsTables:
-             self.log("Creating PSPS table: " + table.name)
+             self.logger.info("Creating PSPS table: " + table.name)
              table.write(self.localUrl + '#' + table.name)
 
@@ -253 +306 @@
     '''   
     def indexIppTables(self):
-        self.log("indexIppTables not implemented")
+        self.logger.warn("indexIppTables not implemented")
 
 
@@ -261 +314 @@
     def createIndex(self, table, column):
 
-        self.log("Creating index on column '"+column+"' for table '"+table+"'")
+        self.logger.info("Creating index on column '"+column+"' for table '"+table+"'")
 
         sql = "CREATE INDEX "+table+"_index ON "+table+" ("+column+")"
@@ -267 +320 @@
             self.localStmt.execute(sql)
         except:
-            self.log("Index already in place on '" + column + "' for table '" + table + "'")
+            self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
 
 
@@ -274 +327 @@
     '''   
     def indexPspsTables(self):
-        self.log("indexPspsTables not implemented")
+        self.logger.warn("indexPspsTables not implemented")
 
     '''
@@ -290 +343 @@
           match = re.match(filter, table.name)
           if not match: continue
-          self.log("Creating IPP table " + table.name)
+          self.logger.info("Creating IPP table " + table.name)
           table = stilts.tpipe(table, cmd='explodeall')
           try:
               table.write(self.localUrl + '#' + table.name)
           except:
-              self.log("ERROR problem writing table '" + table.name + "' to the database")
+              self.logger.exception("Problem writing table '" + table.name + "' to the database")
 
           count = count + 1
 
-      self.log("Imported %d tables from '%s' " % (count, self.inputFitsPath))
+      self.logger.info("Imported %d tables from '%s' " % (count, self.inputFitsPath))
 
       self.indexIppTables()
@@ -308 +361 @@
     def exportPspsTablesToFits(self):
 
-        self.log("Exporting all PSPS tables to FITS")
+        self.logger.info("Exporting all PSPS tables to FITS")
         _tables = []
 
-        self.log("    Selecting database tables")
+        self.logger.info("    Selecting database tables")
         for table in self.pspsTables:
            _table = stilts.tread(self.localUrl + '#SELECT * FROM ' + table.name)
@@ -317 +370 @@
            _tables.append(_table)
 
-        self.log("    Writing to FITS file '" + self.outputFitsPath + "'...")
+        self.logger.info("    Writing to FITS file '" + self.outputFitsPath + "'...")
         stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
-        self.log("    ...done")
+        self.logger.info("    ...done")
 
     '''
@@ -394 +447 @@
     def replaceAllPspsNulls(self, sub):
 
-        self.log("Replacing all NULL values in PSPS tables with '" + sub + "'...")
+        self.logger.info("Replacing all NULL values in PSPS tables with '" + sub + "'...")
         for table in self.pspsTables:
             self.replaceNulls(table.name, sub)
-        self.log("...done")
+        self.logger.info("...done")
 
     '''
@@ -403 +456 @@
     '''
     def populatePspsTables(self):
-        self.log("Not implemented yet")
+        self.logger.warn("Not implemented yet")
 
     '''
@@ -410 +463 @@
     def updateDvoIDs(self, table):
 
-        self.log("Updating table '" + table + "' with DVO IDs...")
+        self.logger.info("Updating table '" + table + "' with DVO IDs...")
         sql = "UPDATE " + table + " AS a, dvo AS b SET \
                a.ippObjID = b.ippObjID, \
@@ -416 +469 @@
                WHERE a.ippDetectID = b.ippDetectID"
         self.localStmt.execute(sql)
-        self.log("...done")
+        self.logger.info("...done")
 
 
@@ -424 +477 @@
     def createDvoTable(self):
 
-        self.log("Creating DVO table for ID matching")
+        self.logger.info("Creating DVO table for ID matching")
         sql = "DROP TABLE dvo"
         self.localStmt.execute(sql)