Changeset 26378

Timestamp:

Dec 9, 2009, 12:14:22 PM (16 years ago)

Author:

watersc1

Message:

dqstatstool: I forgot to commit this last week.

summit copy: added support for use_compress option in the datastore entry in the database. This should allow gpc1 and isp data to be copied using the same code.

dbconfig: database changes to support the above.

Location:

trunk

Files:

• dbconfig/changes.txt (modified) (1 diff)
• dbconfig/dqstats.md (modified) (1 diff)
• dbconfig/summitcopy.md (modified) (1 diff)
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/scripts/dqstats_bundle.pl (added)
• ippScripts/scripts/register_imfile.pl (modified) (2 diffs)
• ippTasks/Makefile.am (modified) (1 diff)
• ippTasks/dqstats.pro (added)
• ippTasks/summit.copy.pro (modified) (3 diffs)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/dqstatstool_createbundle.sql (added)
• ippTools/share/dqstatstool_definebyquery.sql (added)
• ippTools/share/dqstatstool_get_contents.sql (added)
• ippTools/share/dqstatstool_get_run.sql (added)
• ippTools/src/Makefile.am (modified) (5 diffs)
• ippTools/src/dqstatstool.c (added)
• ippTools/src/dqstatstool.h (added)
• ippTools/src/dqstatstoolConfig.c (added)
• ippTools/src/pxdqstats.c (added)
• ippTools/src/pxdqstats.h (added)
• ippTools/src/pztool.c (modified) (1 diff)

trunk/dbconfig/changes.txt

@@ -1466 +1466 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+-- Version 1.1.60
+
+ALTER TABLE dqstatsRun ADD COLUMN label VARCHAR(64) AFTER registered;
+ALTER TABLE dqstatsRun ADD KEY(label);
+
+ALTER TABLE dqstatsRun ADD COLUMN fault SMALLINT NOT NULL AFTER label;
+ALTER TABLE dqstatsRun ADD KEY(fault);
+
+-- Version 1.1.61
+
+ALTER TABLE pzDataStore ADD COLUMN use_compress SMALLINT NOT NULL AFTER epoch;
+
+UPDATE dbversion set schema_version = '1.1.61',  updated= CURRENT_TIMESTAMP();

trunk/dbconfig/dqstats.md

@@ -3 +3 @@
     state          STR          64
     registered     TAI          NULL
+    label          STR          64    # Key
+    fault          S16          0     # Key NOT NULL
 END
 

trunk/dbconfig/summitcopy.md

@@ -4 +4 @@
     uri         STR         255
     epoch       UTC         0001-01-01T00:00:00Z
+    use_compress S16        0
 END
 

trunk/ippScripts/Build.PL

@@ -98 +98 @@
         scripts/rcserver_checkstatus.pl
         scripts/publish_file.pl
+        scripts/dqstats_bundle.pl
         scripts/whichimage
     )],

trunk/ippScripts/scripts/register_imfile.pl

@@ -130 +130 @@
     }
     elsif (($isGPC1 == 1) && ($burntoolState == 1)) {
+#       print STDERR "In the good region: >>$burntoolState<<\n";
         my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -148 +149 @@
         }
         $burntoolState = $burntoolStateGood; # Positive because this has the header table.
+        
     }
     $cmdflags .= " -burntool_state $burntoolState ";

trunk/ippTasks/Makefile.am

@@ -29 +29 @@
         receive.pro \
         publish.pro \
+        dqstats.pro \
         science.cleanup.pro
 

trunk/ippTasks/summit.copy.pro

@@ -147 +147 @@
 # run pzgetexp periodically to populate pzPendingExp in the database (no I/O)
 # this task is querying the data store for a list of exposures ("filesets")
-# and inserting these into a db table on the local cluster (pzPendingExp)
+# and inserting these into a db table on the local cluster (pzDownloadExp)
 task pzgetexp
   periods      -exec     30
@@ -446 +446 @@
         stderr $LOGDIR/summit.copy.log
 
+        book getpage pzDataStore 0 -var PZDSPAGE -key dbname $DBNAME 
+        book getword pzDataStore $PZDSPAGE use_compress  -var USECOMPRESS
+
         $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --dbname $DBNAME --timeout 120 --verbose --copies 2
-        if ($COMPRESS) 
+        if ($USECOMPRESS == 1) 
             $run = $run --compress
-        else
-            $run = $run --bytes $BYTES 
-        end
-        if (("$MD5SUM" != "NULL") && ("$MD5SUM" != "0") && (not($COMPRESS)))
+        end
+#        if (("$MD5SUM" != "NULL") && ("$MD5SUM" != "0") && (not($COMPRESS)))
 # && (($YEAR > 2008) || (("$YEAR" = "2007") && ($MONTH > 8))))
-            $run = $run --md5 $MD5SUM
-        end
+#            $run = $run --md5 $MD5SUM
+#        end
         if ($NEBULOUS) 
             $run = $run --nebulous
@@ -463 +464 @@
         # store the pageName for future reference below
         options $pageName
-
+        
         # create the command line
         if ($VERBOSE > 1)

trunk/ippTools/share/Makefile.am

@@ -140 +140 @@
      disttool_updateinterest.sql \
      disttool_updatercrun.sql \
+     dqstatstool_createbundle.sql \
+     dqstatstool_definebyquery.sql \
+     dqstatstool_get_contents.sql \
+     dqstatstool_get_run.sql \
      faketool_change_exp_state.sql \
      faketool_change_imfile_data_state.sql \

trunk/ippTools/src/Makefile.am

@@ -9 +9 @@
         disttool \
         flatcorr \
+        dqstatstool \
         magictool \
         magicdstool \
@@ -32 +33 @@
         pxdata.h \
         pxfake.h \
+        pxdqstats.h
         pxmagic.h \
         pxregister.h \
@@ -52 +54 @@
         flatcorr.h \
         faketool.h \
+        dqstatstool.h \
         magictool.h \
         magicdstool.h \
@@ -77 +80 @@
         pxfake.c \
         pxfault.c \
+        pxdqstats.c \
         pxmagic.c \
         pxregister.c \
@@ -142 +146 @@
     faketool.c \
     faketoolConfig.c
+
+dqstatstool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+dqstatstool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+dqstatstool_SOURCES = \
+    dqstatstool.c \
+    dqstatstoolConfig.c
 
 magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)

trunk/ippTools/src/pztool.c

@@ -117 +117 @@
             telescope,
             uri,
-            NULL  // epoch
+            NULL,  // epoch
+            0      // use_compress                         
         )) {
         psError(PS_ERR_UNKNOWN, false, "database error");