Changeset 31390

Timestamp:

Apr 27, 2011, 9:50:52 AM (15 years ago)

Author:

rhenders

Message:

working version. loops through all available FITS files and ingest them into a MySQL database

File:

• trunk/ippToPsps/jython/dvoToMySQL.py (modified) (8 diffs)

trunk/ippToPsps/jython/dvoToMySQL.py

@@ -7 +7 @@
 import os
 import logging
+import glob
 from subprocess import call, PIPE, Popen
 
@@ -43 +44 @@
 
         # create DVO tables
-        self.scratchDb.createDvoTables()
+        #self.scratchDb.createDvoTables()
 
         # import Images.dat table
+        sql = "DELETE FROM dvoMetaFull"
+        self.scratchDb.stmt.execute(sql)
+
         imagesTableName = self.importFits(self.pathToDvo, 
                 "", 
@@ -52 +56 @@
         self.scratchDb.createIndex(imagesTableName, "IMAGE_ID")
 
-        # insert into dvoMeta
+        # insert into dvoMetaFull
         self.logger.info("Inserting all image meta data into database")
-        sql = "INSERT INTO dvoMeta ( \
+        sql = "INSERT INTO dvoMetaFull ( \
                sourceID, \
                imageID, \
@@ -66 +70 @@
                FROM " + imagesTableName
         self.scratchDb.stmt.execute(sql)
-
+        
         subdirs = ['n0000']
 
         for subdir in subdirs:
 
-            files = ['0247.06', '0244.06', '0244.10']
+            files = glob.glob(pathToDvo + "/" + subdir + "/*.cpm")
+
+            #files = ['0247.06', '0244.06', '0244.10']
 
             for file in files:
 
-                self.logger.info("---------------------------------------------")
+                # get just filename, without extension
+                file = os.path.basename(os.path.splitext(file)[0])
+                self.logger.info("---------------------------------------------: " + file)
+
+                if self.scratchDb.alreadyImportedThisDvoTable(file): continue
 
                 # import cpm table and index
@@ -112 +122 @@
                 self.scratchDb.stmt.execute(sql)
 
-                # now put everything into dvoDetection table
-                self.logger.info("Putting everything into dvoDetection table")
-                sql = "INSERT INTO dvoDetection (\
+                self.logger.info("Putting everything into dvoDetectionFull table")
+                sql = "INSERT IGNORE INTO dvoDetectionFull (\
                        sourceID \
                        ,imageID \
@@ -137 +146 @@
                 self.scratchDb.dropTable(cpmTableName)
                 self.scratchDb.dropTable(cptTableName)
-        
+       
+                self.scratchDb.setImportedThisDvoTable(file)
+
         self.scratchDb.dropTable(imagesTableName)
 
@@ -176 +187 @@
 
           #try:
+          self.logger.info("Writing FITS table to database")
           table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName)
           #except:
@@ -190 +202 @@
 
 dvoToMySql = DvoToMySql(logger, "/data/ipp005.0/gpc1/catdirs/MD04.merges/MD04.merge")
+#dvoToMySql = DvoToMySql(logger, "/export/ippc00.1/rhenders/MD04.merge")
 
 logger.info("Program complete")