Changeset 35789

Timestamp:

Jul 10, 2013, 12:08:57 PM (13 years ago)

Author:

bills

Message:

add chipRun.update_mode. If set do not use the config dump
file when regenerating images using update

Location:

trunk

Files:

• dbconfig/chip.md (modified) (1 diff)
• ippScripts/scripts/chip_imfile.pl (modified) (10 diffs)
• ippTasks/chip.pro (modified) (2 diffs)
• ippTools/share/chiptool_change_exp_state.sql (modified) (1 diff)
• ippTools/share/chiptool_completely_processed_exp.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (2 diffs)
• ippTools/share/pxadmin_drop_tables.sql (modified) (1 diff)
• ippTools/src/chiptool.c (modified) (2 diffs)
• ippTools/src/chiptoolConfig.c (modified) (2 diffs)
• ippTools/src/pxchip.c (modified) (1 diff)

trunk/dbconfig/chip.md

@@ -20 +20 @@
     maskfrac_magic  F32     0.0
     maskfrac_advisory F32   0.0
+    update_mode S16         0  
     note        STR         255
 END

trunk/ippScripts/scripts/chip_imfile.pl

@@ -45 +45 @@
 # Parse the command-line arguments
 my ( $exp_id, $chip_id, $class_id, $chip_imfile_id, $uri, $camera, $outroot, $dbname, $run_state, $reduction, $threads, $verbose,
-     $no_update, $save_temps, $no_op, $redirect, $magicked, $deburned );
+     $no_update, $save_temps, $no_op, $redirect, $magicked, $deburned, $update_mode );
 
 GetOptions(
@@ -58 +58 @@
     'reduction=s'       => \$reduction, # Reduction class
     'run-state=s'       => \$run_state, # current state of the run (new, update)
+    'update-mode=s'     => \$update_mode, # update_mode if non-zero do not use configdump
     'magicked=s'        => \$magicked,  # magicked state of input file
     'deburned=s'        => \$deburned,  # does deburned image exist?
@@ -147 +148 @@
 my $outputBin2;
 my $dump_config = 1;
-my $do_stats = 1;
+
+# Note: We currently always do_stats because we need the quality flags
+# If run_state eq 'update' using the same file name wipes out the stats from the original processing which
+# is arguably a bug.  Since they are already in the database in that case this probably isn't a big
+# deal
+my $do_stats = 1;   
 $outputStats   = prepare_output("PPIMAGE.STATS",         $outroot, $class_id, 1);
+
 my $do_binned_images = 1;
 
@@ -155 +162 @@
     $configuration = prepare_output("PPIMAGE.CONFIG",        $outroot, $class_id, 1);
 } else {
-    $configuration = $ipprc->filename('PPIMAGE.CONFIG', $outroot, $class_id) 
+    # If update_mode is non-zero we are going to update with current recipes. 
+    # In this case dump the new configuration.
+    if ($update_mode) {
+        $configuration = prepare_output("PPIMAGE.CONFIG",        $outroot, $class_id, 1);
+    } else {
+        $configuration = $ipprc->filename('PPIMAGE.CONFIG', $outroot, $class_id) 
             or &my_die("Missing entry from camera config: PPIMAGE.CONFIG", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-    if ($ipprc->file_exists($configuration)) {
-        $dump_config = 0;
-    } else {
-        print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
-
-        # XXX: should we create a new config dump file?
-        # I vote yes but only if we can distingusing between temporarily unavailable and GONE.
-        my $gone = 0;
-        if (storage_object_exists($configuration, \$gone)) {
-            if ($gone) {
-                $configuration = prepare_output('PPIMAGE.CONFIG', $outroot, $class_id, 1);
-                # if we dump the config we need to insure that the config dump represents
-                # the full processing
-            } else {
-                # file is temporarily not available. Don't dump config.
-                $dump_config = 0;
+        if ($ipprc->file_exists($configuration)) {
+            $dump_config = 0;
+        } else {
+            print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
+
+            # XXX: should we create a new config dump file?
+            # I vote yes but only if we can distingusing between temporarily unavailable and GONE.
+            my $gone = 0;
+            if (storage_object_exists($configuration, \$gone)) {
+                if ($gone) {
+                    $configuration = prepare_output('PPIMAGE.CONFIG', $outroot, $class_id, 1);
+                    # if we dump the config we need to insure that the config dump represents
+                    # the full processing
+                } else {
+                    # file is temporarily not available. Don't dump config.
+                    $dump_config = 0;
+                }
             }
         }
@@ -185 +198 @@
         or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR);
 
-    # don't do binned images when updating unless we are starting from scratch.
-    $do_binned_images = 0 unless $dump_config;
+    # don't do binned images when updating
+    $do_binned_images = 0;
 }
 if ($do_binned_images) {
@@ -219 +232 @@
     my $do_photom = metadataLookupBool($recipeData, 'PHOTOM');
     if ($do_photom and ($run_state eq 'update')) {
-        # If previous photometry outputs are ok skip running photometry
+        # If previous psf file is ok skip photometry
         if ($dump_config || rerun_photometry($outroot, $class_id)) {
             carp "Will rerun photometry\n";
@@ -562 +575 @@
     $command .= " -class_id $class_id";
     $command .= " -dbname $dbname" if defined $dbname;
+    # XXX: if we had a quality problem on this re-run we are not saving that fact in the DB...
+    # to change this we need to modify chiptool -tofullimfile to supprot -set_quality and then add
+    #$command .= " -set_quality $quality" if $quality;
 }
 
@@ -600 +616 @@
 
     my $update_sources_if_gone = 0; # set this to regenerate sources if gone. 
-                                    # We no longer do this as of 2012-12
+                                    # We no longer do this as of 2012-12 since cleanup deletes them
     my $make_sources = 0;
     my $sources_available = 0;
@@ -635 +651 @@
         if (storage_object_exists($outputPsf, \$gone)) {
             # object exists, but no instances are available. If they are permanently gone
-            # rename the storage object
+            # remake it
             if ($gone) {
                 carp "WARNING: PSF storage object exists but all instances are permanently gone";
@@ -641 +657 @@
             }
         } else {
+            # object completely missing remake it
             $make_psf = 1;
         }

trunk/ippTasks/chip.pro

@@ -154 +154 @@
     book getword chipPendingImfile $pageName state -var RUN_STATE
     book getword chipPendingImfile $pageName path_base -var PATH_BASE
+    book getword chipPendingImfile $pageName update_mode -var UPDATE_MODE
 
     if ($RAW_MAGICKED > 0)
@@ -189 +190 @@
       $run = $run --reduction $REDUCTION
     end
+    if ($UPDATE_MODE)
+        $run = $run --update-mode $UPDATE_MODE
+    end
+
     add_standard_args run
 

trunk/ippTools/share/chiptool_change_exp_state.sql

@@ -5 +5 @@
 UPDATE chipRun
 JOIN rawExp using(exp_id)
-    SET chipRun.state = '%s'
+    SET chipRun.state = '%s', chipRun.update_mode = 0
     -- set magicked hook %s
     WHERE

trunk/ippTools/share/chiptool_completely_processed_exp.sql

@@ -15 +15 @@
     end_stage,
     imfile_magicked as magicked,
+    0 as update_mode,
     note
 FROM

trunk/ippTools/share/pxadmin_create_tables.sql

@@ -1 @@
-eREATE TABLE dbversion (
+CREATE TABLE dbversion (
     schema_version VARCHAR(64),
     updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
@@ -299 +299 @@
     maskfrac_magic FLOAT,
     maskfrac_advisory FLOAT,
+    update_mode SMALLINT DEFAULT 0,
     note VARCHAR(255),
     PRIMARY KEY(chip_id),

trunk/ippTools/share/pxadmin_drop_tables.sql

@@ -115 +115 @@
 DROP TABLE IF EXISTS survey;
 DROP TABLE IF EXISTS lapGroup;
+DROP TABLE IF EXISTS mergedvodbCopy;
+DROP TABLE IF EXISTS mergedvodbProcessed;
+DROP TABLE IF EXISTS mergedvodbRun;
 
 SET FOREIGN_KEY_CHECKS=1

trunk/ippTools/src/chiptool.c

@@ -2018 +2018 @@
     PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", false, false);
     PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+    PXOPT_LOOKUP_S16(update_mode, config->args, "-set_update_mode", false, false);
 
     psString query = pxDataGet("chiptool_setimfiletoupdate.sql");
@@ -2029 +2030 @@
         psStringAppend(&setHook, "\n , chipRun.label = '%s'", label);
     }
+    // always set update_mode. If user has not supplied it zero is the right answer.
+    psStringAppend(&setHook, "\n, chipRun.update_mode = %d", update_mode);
 
     if (class_id) {

trunk/ippTools/src/chiptoolConfig.c

@@ -52 +52 @@
 
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir",  0,            "define workdir (required)", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label",  0,            "define label (required)", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label",  0,            "define label", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction",  0,            "define reduction class", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_expgroup",  0,            "define exposure group", NULL);
@@ -107 +107 @@
     psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-class_id", 0,  "search by class_id", NULL);
     psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-set_label",  0,"new value for label", NULL);
+    psMetadataAddS16(setimfiletoupdateArgs, PS_LIST_TAIL, "-set_update_mode",  0,"new value for update_mode", 0);
 
     // -pendingimfile

trunk/ippTools/src/pxchip.c

@@ -283 +283 @@
                        NAN,
                        NAN,
+            0,         // update_mode
             note
             )