Changeset 31201 for trunk/ippToPsps/jython/batch.py

Timestamp:

Apr 6, 2011, 1:07:09 PM (15 years ago)

Author:

rhenders

Message:

added method to create DVO database table and a method to update objIDs for any table with those from DVO table; methiod to get min/max objID from any table; methods to write BatchManifest file in XML

File:

• trunk/ippToPsps/jython/batch.py (modified) (7 diffs)

trunk/ippToPsps/jython/batch.py

@@ -5 +5 @@
 import re
 import sys
+import os
+import md5
 
 from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
 
 from java.lang import *
 from java.sql import *
-from xml.etree.ElementTree import ElementTree
-
+from xml.etree.ElementTree import ElementTree, Element, tostring
+from xml.dom import minidom
 
 '''
@@ -30 +33 @@
 
         # set up class variables
+        self.batchType = batchType;
         self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot"
         self.inputFitsPath = inputFitsPath
@@ -46 +50 @@
         self.localStmt = self.localCon.createStatement()
 
-        # create Gpc1 object
+        # create Gpc1Db object
         self.gpc1Db = Gpc1Db()
-
-        # set up JDBC connection to gpc1 Db
-        dbName = doc.find("gpc1database/name").text
-        dbHost = doc.find("gpc1database/host").text
-        dbUser = doc.find("gpc1database/user").text
-        dbPass = doc.find("gpc1database/password").text
-        self.gpc1Url = "jdbc:mysql://"+dbHost+"/"+dbName+"?user="+dbUser+"&password="+dbPass
-        self.gpc1Con = DriverManager.getConnection(self.localUrl)
-        self.gpc1Stmt = self.localCon.createStatement()
 
         # get survey ID from init table
@@ -67 +62 @@
             self.log("No survey ID found for this survey: '" + survey + "'")
             self.surveyID = -1;
-    
+   
+        # get dvo info from config
+        dvoName = doc.find(survey+"dvo/name").text
+        dvoLocation = doc.find(survey+"dvo/location").text
+         
+        # get datastore info from config
+        datastoreProduct = doc.find("datastore/product").text
+
+        # create IppToPspsDb object and create a new batch
+        self.ippToPspsDb = IppToPspsDb()
+        self.batchID = self.ippToPspsDb.createNewBatch(66, survey, self.getPspsBatchType(), dvoName, datastoreProduct)
+
+        # get local storage location from config
+        self.batchName = "B%08d" % self.batchID
+        self.localOutPath = doc.find("localOutPath").text + "/" + self.getPspsBatchType() + "/" + dvoName + "/%s" % self.batchName 
+        if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath)
+
+
         # store today's date
         now = datetime.datetime.now();
         self.dateStr = now.strftime("%Y-%m-%d")
 
-        if self.inputFitsPath != "": 
-            self.parseFitsHeader()
+        if self.inputFitsPath != "": self.parseFitsHeader()
+
+        # create DVO table
+        self.createDvoTable()
 
     '''
@@ -83 +97 @@
         self.localStmt.close()
         self.localCon.close()
-        self.gpc1Stmt.close()
-        self.gpc1Con.close()
+
+    '''
+    Writes the batch manifest file
+    '''
+    def writeBatchManifest(self):
+
+        outPath = self.localOutPath + "/BatchManifest.xml"
+        tmpPath = "./tmp.xml"
+        self.log("Creating batch manifest file here: " + outPath)
+        root = Element('manifest')
+
+        # batch information
+        root.attrib['name'] = self.batchName
+        root.attrib['type'] = self.getPspsBatchType()
+        root.attrib['survey'] = self.getPspsSurveyType()
+        root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") 
+        root.attrib['minObjId'] = str(self.minObjID)
+        root.attrib['maxObjId'] = str(self.maxObjID)
+
+        # get md5sum
+        fits=os.popen("md5sum " + self.outputFitsPath)
+        for f in fits.readlines(): md5sum = f[0:f.rfind(" ")]
+
+        # get file size
+        fileSize = os.path.getsize(self.outputFitsPath)
+
+        # file information
+        child = Element('file')
+        root.append(child)
+        child.attrib['name'] = self.outputFitsFile
+        child.attrib['bytes'] = str(fileSize)
+        child.attrib['md5'] = md5sum
+
+        # now create doc and write to file
+        file = open(tmpPath, 'w')
+        ElementTree(root).write(file)
+        file.close()
+
+        # clunky way to prettify XML
+        os.popen("xmllint --format " + tmpPath + " > " + outPath)
+#        os.remove(tmpPath)
+
+
+    '''
+    Gets PSPS-friendly survey type
+    '''
+    def getPspsSurveyType(self):
+
+        if self.survey == "ThreePi": return "3PI"
+        elif self.survey == "MD04": return "MD04"
+        else: self.log("ERROR: Don't know this survey: " + self.survey)
+
+    '''
+    Gets PSPS friendly batch type
+    '''
+    def getPspsBatchType(self):
+
+        if self.batchType == "init": return "IN"
+        elif self.batchType == "detection": return "P2"
+        elif self.batchType == "stack": return "ST"
+        else: self.log("ERROR: Don't know this batch type: " + self.survey)
+
 
     '''
@@ -92 +166 @@
 
         print datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " | " + msg
+
+    '''
+    Sets min and max obj ID using the provided table
+    '''
+    def setMinMaxObjID(self, table):
+
+        sql = "SELECT MIN(objID), MAX(objID) FROM " + table
+        print 
+        rs = self.localStmt.executeQuery(sql)
+        rs.first()
+        self.minObjID = rs.getLong(1)
+        self.maxObjID = rs.getLong(2)
+
+        print "MIN mAX = %d %d" % (self.minObjID,  self.maxObjID)
 
     '''
@@ -317 +405 @@
         self.log("Not implemented yet")
 
+    '''
+    Updates provided table with DVO IDs from DVO table
+    '''
+    def updateDvoIDs(self, table):
+
+        self.log("Updating table '" + table + "' with DVO IDs...")
+        sql = "UPDATE " + table + " AS a, dvo AS b SET \
+               a.ippObjID = b.ippObjID, \
+               a.objID = b.objID \
+               WHERE a.ippDetectID = b.ippDetectID"
+        self.localStmt.execute(sql)
+        self.log("...done")
+
+
+    '''
+    Creates a table for for ID matching
+    '''
+    def createDvoTable(self):
+
+        self.log("Creating DVO table for ID matching")
+        sql = "DROP TABLE dvo"
+        self.localStmt.execute(sql)
+        sql = "CREATE TABLE dvo (ippDetectID BIGINT PRIMARY KEY, ippObjID BIGINT, objID BIGINT)"
+        self.localStmt.execute(sql)
+