Changeset 35097 for trunk/ippToPsps/jython/gpc1db.py

Timestamp:

Feb 6, 2013, 3:16:35 PM (13 years ago)

Author:

eugene

Message:

upgrade to ippToPsps (see doc/upgrade.txt): adds native dvo to mysql ingest operations, adds autogen configuration & installation, splits out global config information from new "skychunk" information (current region on the sky being processed), adds test suites

Location:

trunk/ippToPsps

Files:

• . (modified) (1 prop)
• jython (modified) (1 prop)
• jython/gpc1db.py (modified) (6 diffs)

trunk/ippToPsps/jython

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in

trunk/ippToPsps/jython/gpc1db.py

@@ -21 +21 @@
     '''
     def __init__(self, logger, config):
-        super(Gpc1Db, self).__init__(logger, config, "gpc1database")
+        # define database type
+        if (config.test): 
+            dbType = "gpc1database_test"
+        else:
+            dbType = "gpc1database"
+
+        super(Gpc1Db, self).__init__(logger, config, dbType)
 
     '''
@@ -93 +99 @@
             
         try:
+            # XXX EAM : test output
+            self.logger.infoPair("sql for dvo items:", sql)
             rs = self.executeQuery(sql)
             while (rs.next()): 
@@ -112 +120 @@
         self.logger.debug("Querying GPC1 for image IDs for stack ID: " + str(stackID))
 
+        # JOIN stackInputSkyfile shoule use stack_id as well as warp_id, right? (maybe not -- the join to warp is first)
         sql = "SELECT DISTINCT CONCAT(exp_id, SUBSTR(class_id, 3, 4)) FROM ( \
                SELECT DISTINCT exp_id,class_id \
@@ -226 +235 @@
         if path.startswith("neb"):
 
+            # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
             f=os.popen("neb-ls -p "+path+"%smf")
             for i in f.readlines():
@@ -234 +244 @@
             files = glob.glob(path + ".smf")
 
+        # XXX EAM : test output
+        self.logger.infoPair("smf files:", files)
         if len(files) < 1: return None
 
@@ -290 +302 @@
 
 
+        # print "staring stack stage cmf"
+
         # get single path base for the directory containing all cmf files, one of which is hopefully for our stack_id
         try:
             rs = self.executeQuery(sql)
             rs.first()
+        except:
+            self.logger.errorPair("Can't query for stack cmfs", "stack_id = %d" % stackID)
+            return None
+
+        try:
             pathBase = rs.getString(1)
+        except:
+            self.logger.errorPair("No stack cmf files found for stack_id = %d" % stackID)
+            return None
+
+        # find the files (are they in nebulous or in the filesystem?)
+        files = []
+        if pathBase.startswith("neb"):
+            # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
+            f=os.popen("neb-ls -p " + pathBase + "%cmf")
+            for i in f.readlines():
+                files.append(i.rstrip())
+
+        # or not a neb path
+        else:
+            files = glob.glob(pathBase + ".cmf")
+
+        # print "stack cmf files:", files
+        if len(files) < 1: return None
+
+        # if we get here, then the cmf is readable, now check the stack_id
+        fits = Fits(self.logger, self.config, files[0])
+        return fits
+
+        # XXX validate the file?
+        # if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
 
             # now find and loop through all cmf files at this path_base
-            files=os.popen("neb-ls " + pathBase + "%cmf")
-            for i in files.readlines():
-
-                nebPath = i.rstrip()
-
-                # now attempt to run 'neb-ls -p' to actually read this cmf file
-                try:
-                    self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'")
-                    paths=os.popen("neb-ls -p " + nebPath)
-                    path = paths.readline().rstrip()
-                    if len(path) < 1:
-                       self.logger.debug("zero length path - skipping")
-                       raise
-
-                    # if we get here, then the cmf is readable, now check the stack_id
-                    fits = Fits(self.logger, self.config, path)
-                    if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
-                        # we have the right file!
-                        self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
-                        return fits
-                    else:
-                        # this is not the correct file
-                        self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
-
-                # an exception here means that nebulous could not read this file, so we just skip it
-                except:
-                    self.logger.debug("NEB FAILED SKIPPING")
-                    pass
-
-        except:
-            self.logger.errorPair("Can't query for stack cmfs", "stack_id = %d" % stackID)
-
-        self.logger.errorPair("Couldn't find stack cmf for", "stack_id = %d" % stackID)
-        return None
+
+            ## XXX the code below was needed when we ingested from staticsky (N files per run)
+            ## XXX ingesting from skycal solves this problem (1 file per run)
+            ## files=os.popen("neb-ls " + pathBase + "%cmf")
+            ## for i in files.readlines():
+            ##     nebPath = i.rstrip()
+            ## 
+            ##     # now attempt to run 'neb-ls -p' to actually read this cmf file
+            ##     try:
+            ##         self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'")
+            ##         paths=os.popen("neb-ls -p " + nebPath)
+            ##         path = paths.readline().rstrip()
+            ##         if len(path) < 1:
+            ##            self.logger.debug("zero length path - skipping")
+            ##            raise
+            ## 
+            ##         print "starint stack stage cmf 3"
+            ## 
+            ##         # if we get here, then the cmf is readable, now check the stack_id
+            ##         fits = Fits(self.logger, self.config, path)
+            ##         if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
+            ##             # we have the right file!
+            ##             self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
+            ##             return fits
+            ##         else:
+            ##             # this is not the correct file
+            ##             self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
+            ## 
+            ##     # an exception here means that nebulous could not read this file, so we just skip it
+            ##     except:
+            ##         self.logger.debug("NEB FAILED SKIPPING")
+            ##         pass
 
     '''

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in