Changeset 35217

Timestamp:

Mar 1, 2013, 1:27:47 PM (13 years ago)

Author:

eugene

Message:

fix the plotter stuff (HAF); use boxes scaled by cos(DEC); use ranges for ra float comparisons; use installed perl programs (in PATH); do not attempt to re-publish automatically

Location:

trunk/ippToPsps

Files:

• config/settings.xml (modified) (3 diffs)
• jython/config.py (modified) (2 diffs)
• jython/dvo.py (modified) (1 diff)
• jython/dxlayer.py (modified) (1 diff)
• jython/ipptopspsdb.py (modified) (2 diffs)
• jython/metrics.py (modified) (4 diffs)
• jython/odm.py (modified) (1 diff)
• jython/plot.py (modified) (3 diffs)
• jython/plotter.py (modified) (1 diff)
• jython/queue.py (modified) (1 diff)

trunk/ippToPsps/config/settings.xml

@@ -86 +86 @@
   </localdatabase_test>
 
-  <!-- GPC1 Db section (TEST VERSION) -->
+  <!-- GPC1 Db section (REAL VERSION) -->
+  <gpc1database_test>
+    <name>gpc1</name>
+    <host>ippdb01</host>
+    <user>ipp</user>
+    <password>ipp</password>
+  </gpc1database_test>
+
+  <!-- GPC1 Db section (TEST VERSION)
   <gpc1database_test>
     <name>gpc1test</name>
@@ -92 +100 @@
     <user>dvo</user>
     <password>dvo</password>
-  </gpc1database_test>
+  </gpc1database_test> -->
 
   <!-- ippToPsps Db section (TEST VERSION)  -->
@@ -105 +113 @@
   <czardatabase>
     <name>czardb</name>
-    <host>none</host>
+    <host>ippdb01</host>
     <user>ipp</user>
     <password>ipp</password>

trunk/ippToPsps/jython/config.py

@@ -58 +58 @@
         self.settingsDoc = ElementTree(file=self.settingsPath)
         self.logPath = self.settingsDoc.find("logPath").text
+        self.czarPlotsPath = self.settingsDoc.find("czarPlotsPath").text
+
+
 
         print "config.programName: ", self.programName
@@ -64 +67 @@
         print "config.logPath: ", self.logPath
         print "config.test: ", self.test
+        print "config.czarPlotsPath: ", self.czarPlotsPath
 
         root = self.settingsDoc.getroot()

trunk/ippToPsps/jython/dvo.py

@@ -6 +6 @@
 import sys
 import os
+import math
 import logging
 import glob

trunk/ippToPsps/jython/dxlayer.py

@@ -30 +30 @@
         tempFile = tempfile.NamedTemporaryFile(mode='w+b')
 
-        p = Popen("../perl/deleteFromDXLayer.pl -b " + batch + " -f " + tempFile.name, 
+        p = Popen("deleteFromDXLayer.pl -b " + batch + " -f " + tempFile.name, 
                 shell=True, 
                 stdout=PIPE)

trunk/ippToPsps/jython/ipptopspsdb.py

@@ -1214 +1214 @@
                 self.logger.errorPair("No outstanding boxes to steal", "no stripe obtained")
 
+        # tests against ra_center need to use a finite box
+        minRA = raCenter - 0.01
+        maxRA = raCenter + 0.01
+
         self.logger.infoPair("Using RA stripe", "%f" % raCenter)
         sql = "SELECT DISTINCT id \
                FROM box \
                JOIN pending ON (id = box_id) \
-               WHERE ra_center = " + str(raCenter) + " \
+               WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
                AND batch_type = '" + batchType + "' \
                AND skychunk = '" + self.skychunk.name + "' \
@@ -1280 +1284 @@
                 self.logger.errorPair("No outstanding boxes to steal", "no stripe obtained")
 
+        # tests against ra_center need to use a finite box
+        minRA = raCenter - 0.01
+        maxRA = raCenter + 0.01
+
         self.logger.infoPair("Using RA stripe", "%f" % raCenter)
         sql = "SELECT DISTINCT id \
                FROM box \
                JOIN pending ON (id = box_id) \
-               WHERE ra_center = " + str(raCenter) + " \
+               WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
                AND skychunk = '" + self.skychunk.name + "' \
                ORDER BY dec_center DESC"

trunk/ippToPsps/jython/metrics.py

@@ -25 +25 @@
     
         self.czarDb = CzarDb(self.logger, self.config)
-        self.plot = Plot(self.logger, self.config, self.ippToPspsDb)
+        self.plot = Plot(self.logger, self.skychunk, self.config, self.ippToPspsDb)
    
     '''
@@ -37 +37 @@
             now = datetime.datetime.now()
             self.logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
-            self.logger.infoPair("Config", self.config.name)
-            self.logger.infoPair("Loading epoch", self.config.epoch)
-            self.logger.infoPair("DVO label", self.config.dvoLabel)
-            self.config.printBoxCoords()
+            self.logger.infoPair("Config", self.skychunk.name)
+            self.logger.infoPair("Loading epoch", self.skychunk.epoch)
+            self.logger.infoPair("DVO label", self.skychunk.dvoLabel)
+            self.skychunk.printBoxCoords()
         
             self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
@@ -77 +77 @@
        
             self.checkClientStatus()
-
+            self.logger.infoPair("here","yay")
             # create a density plot of pending stuff for czartool
             self.plot.createDensityPlot("P2", True)
             self.plot.createDensityPlot("ST", True)
             self.plot.createDensityPlot("OB", True)
-
+            self.logger.infoPair("here","yay2")
             # some clean-up before going to sleep
             self.ippToPspsDb.purgeDeadClients()
@@ -132 +132 @@
            prevList = success
     
-           self.czarDb.insertStats(stage, self.config.name, batchType, numPending, numSuccess, numFail)
+           self.czarDb.insertStats(stage, self.skychunk.name, batchType, numPending, numSuccess, numFail)
     
            if stage == 'processed': numPendingProcessed = numPending

trunk/ippToPsps/jython/odm.py

@@ -32 +32 @@
         tempFile = tempfile.NamedTemporaryFile(mode='w+b')
 
-        p = Popen("../perl/checkOdmStatus.pl -b " + batch + " -f " + tempFile.name, 
+        p = Popen("checkOdmStatus.pl -b " + batch + " -f " + tempFile.name, 
                 shell=True, 
                 stdout=PIPE,

trunk/ippToPsps/jython/plot.py

@@ -15 +15 @@
     Constructor
     '''
-    def __init__(self, logger, skychunk, ippToPspsDb):
+    def __init__(self, logger, skychunk, config, ippToPspsDb):
 
         self.logger = logger
         self.skychunk = skychunk 
+        self.config = config
         self.ippToPspsDb = ippToPspsDb
 
@@ -25 +26 @@
     '''
     def createDensityPlot(self, batchType, forCzartool=False):
-
+        
         tempFilename = "./" + self.skychunk.name + "_" + batchType + "_plotData.dat"
         DATFILE = open(tempFilename,'w')
         max = self.ippToPspsDb.createPendingDensityPlotData(batchType, DATFILE)
         DATFILE.close()
-
         if max == 0: max = 1
 
         timestamp = time.strftime('%Y_%m%d_%H%M%S')
-
+        
         if forCzartool:
             ## XXX from config??
-            OUTPUTFILE = self.skychunk.czarPlotsPath + "/ippToPsps_density_" + self.skychunk.name + "_" + batchType + ".png"
+            OUTPUTFILE = self.config.czarPlotsPath + "/ippToPsps_density_" + self.skychunk.name + "_" + batchType + ".png"
         else:
             OUTPUTFILE = self.skychunk.name + "_" + batchType + "_" + timestamp + ".png"
-
+            
         f = os.popen('/home/panstarrs/ipp/local/bin/gnuplot', 'w')
-            
+        
         if 0:
             TERM = "X11"
         else:
             TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8"
-                     
+        print "output to : "+ OUTPUTFILE     
         print >> f, "set term " + TERM + "; \
             set output \"" + OUTPUTFILE + "\"; \
@@ -57 +57 @@
             set palette rgb 22,13,10; \
             set ylabel \"Dec\"; \
-            set ylabel \"RA\"; \
+            set xlabel \"RA\"; \
             set cbrange [0:" + str(max) + "]; \
             set datafile missing \"NaN\"; \

trunk/ippToPsps/jython/plotter.py

@@ -18 +18 @@
         super(Plotter, self).__init__(argv)
 
-        self.plot = Plot(self.logger, self.skychunk, self.ippToPspsDb)
+        self.plot = Plot(self.logger, self.skychunk, self.config, self.ippToPspsDb)
  
         if len(argv) < 3:

trunk/ippToPsps/jython/queue.py

@@ -57 +57 @@
             for batchType in self.skychunk.batchTypes: 
 
-                self.logger.infoTitle("Previous failed datastore loads")
-                self.publishAnyUnpublishedBatches(batchType)
+                # self.logger.infoTitle("Previous failed datastore loads")
+                # self.publishAnyUnpublishedBatches(batchType)
 
                 # get totals for whole area to check if there is anything to do