Index: trunk/ippToPsps/jython/batch.py =================================================================== --- trunk/ippToPsps/jython/batch.py (revision 31201) +++ trunk/ippToPsps/jython/batch.py (revision 31222) @@ -7,5 +7,9 @@ import os import md5 - +import shutil +import logging +from subprocess import call, PIPE, Popen + +from datastore import Datastore from gpc1db import Gpc1Db from ipptopspsdb import IppToPspsDb @@ -14,5 +18,4 @@ from java.sql import * from xml.etree.ElementTree import ElementTree, Element, tostring -from xml.dom import minidom ''' @@ -30,5 +33,9 @@ "../config/2/tables.vot" ''' - def __init__(self, batchType, inputFitsPath="", survey=""): + def __init__(self, logger, batchType, inputFitsPath="", survey=""): + + # set up logging + self.logger = logger + self.logger.debug("Batch class constructor") # set up class variables @@ -51,5 +58,5 @@ # create Gpc1Db object - self.gpc1Db = Gpc1Db() + self.gpc1Db = Gpc1Db(self.logger) # get survey ID from init table @@ -60,21 +67,30 @@ self.surveyID = rs.getInt(1) except: - self.log("No survey ID found for this survey: '" + survey + "'") + self.logger.exception("No survey ID found for this survey: '" + survey + "'") self.surveyID = -1; # get dvo info from config - dvoName = doc.find(survey+"dvo/name").text - dvoLocation = doc.find(survey+"dvo/location").text + if survey != "": + dvoName = doc.find(survey+"dvo/name").text + dvoLocation = doc.find(survey+"dvo/location").text + else: + dvoName = "" + dvoLocation = "" # get datastore info from config - datastoreProduct = doc.find("datastore/product").text + self.datastore = Datastore(self.logger) # create IppToPspsDb object and create a new batch - self.ippToPspsDb = IppToPspsDb() - self.batchID = self.ippToPspsDb.createNewBatch(66, survey, self.getPspsBatchType(), dvoName, datastoreProduct) + self.ippToPspsDb = IppToPspsDb(logger) + self.batchID = self.ippToPspsDb.createNewBatch(66, + survey, + self.getPspsBatchType(), + dvoName, + self.datastore.product) # get local storage location from config self.batchName = "B%08d" % self.batchID - self.localOutPath = doc.find("localOutPath").text + "/" + self.getPspsBatchType() + "/" + dvoName + "/%s" % self.batchName + self.subDir = doc.find("localOutPath").text + "/" + self.getPspsBatchType() + "/" + dvoName + self.localOutPath = self.subDir + "/" + self.batchName if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath) @@ -94,5 +110,5 @@ def __del__(self): - self.log("Batch destructor") + self.logger.debug("Batch destructor") self.localStmt.close() self.localCon.close() @@ -105,5 +121,5 @@ outPath = self.localOutPath + "/BatchManifest.xml" tmpPath = "./tmp.xml" - self.log("Creating batch manifest file here: " + outPath) + self.logger.info("Creating batch manifest file here: " + outPath) root = Element('manifest') @@ -117,6 +133,8 @@ # get md5sum - fits=os.popen("md5sum " + self.outputFitsPath) - for f in fits.readlines(): md5sum = f[0:f.rfind(" ")] + p = Popen("md5sum " + self.outputFitsPath, shell=True, stdout=PIPE) + p.wait() + out = p.stdout.read() + md5sum = out[0:out.rfind(" ")] # get file size @@ -136,7 +154,42 @@ # clunky way to prettify XML - os.popen("xmllint --format " + tmpPath + " > " + outPath) -# os.remove(tmpPath) - + p = Popen("xmllint --format " + tmpPath + " > " + outPath, shell=True, stdout=PIPE) + p.wait() + os.remove(tmpPath) + + + ''' + tar and zips batch directory + ''' + def createTarball(self): + + # set up filenams and paths + tarFile = self.batchName + ".tar" + tarPath = self.subDir + "/" + tarFile + + self.tarballFile = tarFile + ".gz" + tarballPath = self.subDir + "/" + self.tarballFile + + # tar directory + p = Popen("tar -cvf " + tarPath + "\ + -C " + self.subDir + " \ + " + self.batchName, shell=True, stdout=PIPE) + p.wait() + + # zip tar archive + p = Popen("gzip -c " + tarPath + " > " + tarballPath, shell=True, stdout=PIPE) + p.wait() + + # delete tar file and original directory + os.remove(tarPath) + shutil.rmtree(self.localOutPath) + + ''' + Publishes this batch to the datastore + ''' + def publishToDatastore(self): + + self.datastore.publish(self.batchName, self.subDir, self.tarballFile, "tgz") + # TODO update ippToPsps Db here ''' @@ -147,5 +200,5 @@ if self.survey == "ThreePi": return "3PI" elif self.survey == "MD04": return "MD04" - else: self.log("ERROR: Don't know this survey: " + self.survey) + else: self.logger.error("Don't know this survey: " + self.survey) ''' @@ -157,5 +210,5 @@ elif self.batchType == "detection": return "P2" elif self.batchType == "stack": return "ST" - else: self.log("ERROR: Don't know this batch type: " + self.survey) + else: self.logger.error("Don't know this batch type: " + self.survey) @@ -244,5 +297,5 @@ self.pspsTables = stilts.treads(self.pspsVoTableFilePath) for table in self.pspsTables: - self.log("Creating PSPS table: " + table.name) + self.logger.info("Creating PSPS table: " + table.name) table.write(self.localUrl + '#' + table.name) @@ -253,5 +306,5 @@ ''' def indexIppTables(self): - self.log("indexIppTables not implemented") + self.logger.warn("indexIppTables not implemented") @@ -261,5 +314,5 @@ def createIndex(self, table, column): - self.log("Creating index on column '"+column+"' for table '"+table+"'") + self.logger.info("Creating index on column '"+column+"' for table '"+table+"'") sql = "CREATE INDEX "+table+"_index ON "+table+" ("+column+")" @@ -267,5 +320,5 @@ self.localStmt.execute(sql) except: - self.log("Index already in place on '" + column + "' for table '" + table + "'") + self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'") @@ -274,5 +327,5 @@ ''' def indexPspsTables(self): - self.log("indexPspsTables not implemented") + self.logger.warn("indexPspsTables not implemented") ''' @@ -290,14 +343,14 @@ match = re.match(filter, table.name) if not match: continue - self.log("Creating IPP table " + table.name) + self.logger.info("Creating IPP table " + table.name) table = stilts.tpipe(table, cmd='explodeall') try: table.write(self.localUrl + '#' + table.name) except: - self.log("ERROR problem writing table '" + table.name + "' to the database") + self.logger.exception("Problem writing table '" + table.name + "' to the database") count = count + 1 - self.log("Imported %d tables from '%s' " % (count, self.inputFitsPath)) + self.logger.info("Imported %d tables from '%s' " % (count, self.inputFitsPath)) self.indexIppTables() @@ -308,8 +361,8 @@ def exportPspsTablesToFits(self): - self.log("Exporting all PSPS tables to FITS") + self.logger.info("Exporting all PSPS tables to FITS") _tables = [] - self.log(" Selecting database tables") + self.logger.info(" Selecting database tables") for table in self.pspsTables: _table = stilts.tread(self.localUrl + '#SELECT * FROM ' + table.name) @@ -317,7 +370,7 @@ _tables.append(_table) - self.log(" Writing to FITS file '" + self.outputFitsPath + "'...") + self.logger.info(" Writing to FITS file '" + self.outputFitsPath + "'...") stilts.twrites(_tables, self.outputFitsPath, fmt='fits') - self.log(" ...done") + self.logger.info(" ...done") ''' @@ -394,8 +447,8 @@ def replaceAllPspsNulls(self, sub): - self.log("Replacing all NULL values in PSPS tables with '" + sub + "'...") + self.logger.info("Replacing all NULL values in PSPS tables with '" + sub + "'...") for table in self.pspsTables: self.replaceNulls(table.name, sub) - self.log("...done") + self.logger.info("...done") ''' @@ -403,5 +456,5 @@ ''' def populatePspsTables(self): - self.log("Not implemented yet") + self.logger.warn("Not implemented yet") ''' @@ -410,5 +463,5 @@ def updateDvoIDs(self, table): - self.log("Updating table '" + table + "' with DVO IDs...") + self.logger.info("Updating table '" + table + "' with DVO IDs...") sql = "UPDATE " + table + " AS a, dvo AS b SET \ a.ippObjID = b.ippObjID, \ @@ -416,5 +469,5 @@ WHERE a.ippDetectID = b.ippDetectID" self.localStmt.execute(sql) - self.log("...done") + self.logger.info("...done") @@ -424,5 +477,5 @@ def createDvoTable(self): - self.log("Creating DVO table for ID matching") + self.logger.info("Creating DVO table for ID matching") sql = "DROP TABLE dvo" self.localStmt.execute(sql)