Changeset 23342

Timestamp:

Mar 17, 2009, 10:36:05 AM (17 years ago)

Author:

eugene

Message:

verify vs master: verify needs to use the detNormalizedImfile entry / master uses detStackedImfile; verify should not block on detNormalizedExp; verify does not select normalization values

Location:

trunk

Files:

• ippScripts/scripts/detrend_resid_exp.pl (modified) (8 diffs)
• ippTasks/detrend.resid.pro (modified) (1 diff)
• ippTools/share/dettool_toresidexp.sql (modified) (4 diffs)

trunk/ippScripts/scripts/detrend_resid_exp.pl

@@ -45 +45 @@
 
 # parse the command-line options
-my ( $det_id, $iter, $exp_id, $exp_tag, $det_type, $camera, $filter, $reject, $outroot, $dbname, $reduction,
+my ( $det_id, $iter, $exp_id, $exp_tag, $det_mode, $det_type, $camera, $filter, $reject, $outroot, $dbname, $reduction,
      $verbose, $no_update, $no_op, $save_temps, $redirect );
 GetOptions(
@@ -53 +53 @@
            'exp_tag|=s'        => \$exp_tag,
            'det_type|t=s'      => \$det_type,
+           'det_mode=s'        => \$det_mode,
            'camera=s'          => \$camera,
            'outroot|w=s'       => \$outroot,   # output file base name
@@ -74 +75 @@
     defined $exp_tag  and
     defined $det_type and
+    defined $det_mode and
     defined $camera   and
     defined $outroot;
@@ -93 +95 @@
 # Get list of normalizations by class_id : stored as $norms; save to temp file for ppImage runs below
 my (%norms, $normsName);
-{
+if ($det_mode eq 'master') {
     # dettool command to select imfile data for this exp_id
     $command  = "$dettool -normalizedstat";
@@ -164 +166 @@
     my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.stats.XXXX", UNLINK => !$save_temps );
 
+    print $statFile "rawResidImfile MULTI\n";
+
     # parse the file info in the metadata
     # as we parse the list of files and their stats, apply the normalization to the relevant fields
@@ -188 +192 @@
         # modify and save the data in this block:
         foreach my $data (@$mdComponents) {
-            my $norm = $norms{$class_id};
+            my $norm = 1.0;
+            if ($det_mode eq "master") {
+                $norm = $norms{$class_id};
+            }
 
             # fields to modify by the normalization:
@@ -252 +259 @@
     $command .= " -recipe PPIMAGE PPIMAGE_J1";
     $command .= " -recipe JPEG $recipe";
-    $command .= " -normlist $normsName";
+    $command .= " -normlist $normsName" if defined $normsName;
     $command .= " -dbname $dbname" if defined $dbname;
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -267 +274 @@
     $command .= " -recipe PPIMAGE PPIMAGE_J2";
     $command .= " -recipe JPEG $recipe";
-    $command .= " -normlist $normsName";
+    $command .= " -normlist $normsName" if defined $normsName;
     $command .= " -dbname $dbname" if defined $dbname;
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

trunk/ippTasks/detrend.resid.pro

@@ -316 +316 @@
     stderr $LOGDIR/detrend.resid.exp.log
 
-    $run = detrend_resid_exp.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose
+    $run = detrend_resid_exp.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_mode $MODE --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose
 
     add_standard_args run

trunk/ippTools/share/dettool_toresidexp.sql

@@ -20 +20 @@
 -- detResidImfile.{det_id, iteration, exp_id} is not in detResidExp
 
-SELECT DISTINCT
+SELECT
     det_id,
     iteration,
@@ -30 +30 @@
     workdir,
     exp_tag
-FROM
-    (SELECT DISTINCT
+FROM 
+(   SELECT
         detRun.det_id AS det_id,
         detRun.iteration,
         detRun.det_type,
         detRun.mode,
-        detRun.workdir,
         detInputExp.exp_id,
         detInputExp.include,
         rawExp.camera,
-        rawExp.exp_tag,
-        detResidImfile.class_id
+        detRun.workdir,
+        rawExp.exp_tag
     FROM detRun
     JOIN detNormalizedExp
@@ -62 +61 @@
     WHERE
         detRun.state = 'run'
+        AND detRun.mode = 'master'
         AND detResidExp.det_id IS NULL
         AND detResidExp.iteration IS NULL
@@ -72 +72 @@
         COUNT(rawImfile.class_id) = COUNT(detResidImfile.class_id)
         AND SUM(detResidImfile.fault) = 0
-    ) AS toresidexp
+    UNION
+    SELECT
+        detRun.det_id AS det_id,
+        detRun.iteration,
+        detRun.det_type,
+        detRun.mode,
+        detInputExp.exp_id,
+        detInputExp.include,
+        rawExp.camera,
+        detRun.workdir,
+        rawExp.exp_tag
+    FROM detRun
+    JOIN detInputExp
+        USING(det_id, iteration)
+    JOIN rawExp
+        ON detInputExp.exp_id = rawExp.exp_id
+    JOIN rawImfile
+        on rawExp.exp_id = rawImfile.exp_id
+    LEFT JOIN detResidImfile
+        ON detRun.det_id = detResidImfile.det_id
+        AND detRun.iteration = detResidImfile.iteration
+        AND detInputExp.exp_id = detResidImfile.exp_id
+        AND rawImfile.class_id = detResidImfile.class_id
+    LEFT JOIN detResidExp
+        ON detResidImfile.det_id = detResidExp.det_id
+        AND detResidImfile.iteration = detResidExp.iteration
+        AND detResidImfile.exp_id = detResidExp.exp_id
+    WHERE
+        detRun.state = 'run'
+        AND detRun.mode = 'verify'
+        AND detResidExp.det_id IS NULL
+        AND detResidExp.iteration IS NULL
+        AND detResidExp.exp_id IS NULL
+    GROUP BY
+        detInputExp.exp_id,
+        detRun.iteration,
+        detRun.det_id
+    HAVING
+        COUNT(rawImfile.class_id) = COUNT(detResidImfile.class_id)
+        AND SUM(detResidImfile.fault) = 0
+) as temp