Changeset 31940

Timestamp:

Jul 26, 2011, 3:29:36 PM (15 years ago)

Author:

watersc1

Message:

Bugfix to allow SAS.12 gri data to process correctly without failing repeatedly at the stack queue stage.

File:

• tags/ipp-20110622/ippScripts/scripts/lap_science.pl (modified) (14 diffs)

tags/ipp-20110622/ippScripts/scripts/lap_science.pl

@@ -29 +29 @@
 my ( $lap_id );
 my ( $chip_mode, $monitor_mode, $cleanup_mode);
+
+# Global configuration constants that probably should be read from elsewhere.
+my $qstack_threshold = 0.05; # Only make a quickstack if more than 5% of the exposures require it.
 
 GetOptions(
@@ -85 +88 @@
     unless (($lapRunInfo{state} eq 'full')||
             ($lapRunInfo{state} eq 'drop')) {
-        &my_die("Cannot run cleanup_mode if lapRun != done!", $lap_id);
+        &my_die("Cannot run cleanup_mode if lapRun != full!", $lap_id);
     }
     my $status = cleanup_mode($lap_id);
@@ -456 +459 @@
     my $can_fstack = 0;
     my $total_exposures = 0;
+    my @lonely_exposures = ();
     foreach my $exposure (@$exposures) {
         $total_exposures++;
@@ -473 +477 @@
         
         if ($exposure->{private}) { # I've declared this exposure private to this lapRun.
-            $needs_qstack = 1;
+            $needs_qstack++;
+            push @lonely_exposures, $exposure;
+#           $lonely_exposures++;
         }
         
@@ -510 +516 @@
             unless ((defined($exposure->{diffRun_state}))&&
                     ($exposure->{diffRun_state} eq 'full')) {
-                $needs_qstack = 1;
+                $needs_qstack++;
                 $needs_something_private = 1;
                 if ($companion) {
@@ -516 +522 @@
                     $companion->{pairwise} = 0;
                     &update_this_exposure($companion);
+                    push @lonely_exposures, $companion;
+#                   $lonely_exposures++;
                 }
                 $exposure->{private} = 1;
@@ -566 +574 @@
         exit(0);
     }
-    if (($needs_qstack == 1)&&              # Do we need the quick stack?
+    if (($needs_qstack > 0)&&              # Do we need the quick stack?
         ($defined_qstack == 0)&&            # Have we not made it already?
         ($can_qstack == $total_exposures)&& # Are all warps done?
         ($needs_something_remade == 0)      # Do we need to wait for a stolen exposure?
         ) {
-        print "STATUS: Will now queue quickstacks\n";
-        &queue_quickstack($exposures);
+        if ($needs_qstack < $total_exposures * $qstack_threshold) { # Drop any exposures that are requiring the qstack
+            print "STATUS: Insufficient unpaired exposures to justify quickstack.  Dropping unpaired exposures\n";
+            foreach my $exposure (@lonely_exposures) {
+                $exposure->{data_state} = 'drop';
+                &update_this_exposure($exposure);
+            }
+        }
+        else {
+            print "STATUS: Will now queue quickstacks\n";
+            &queue_quickstack($exposures);
+        }
     }
 
@@ -623 +640 @@
     my $warps = '';
     foreach $exposure (@$exposures) {
-        if (($exposure->{data_state} != 'drop')&&
+        if (($exposure->{data_state} ne 'drop')&&
             (S64_IS_NOT_NULL($exposure->{warp_id}))) {
             $warps .= " -warp_id $exposure->{warp_id} ";
@@ -640 +657 @@
     $command .= " -pretend " if defined $debug;
     $command .= " -dbname $dbname " if defined $dbname;
-    $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter ";
+    $command .= " -definebyquery -select_label ${label}% -select_skycell_id ${proj_cell}.% -select_filter $filter ";
     $command .= " -set_label ${label} -set_data_group $data_group ";
     $command .= "  -set_workdir $workdir  -set_dist_group NODIST ";
@@ -702 +719 @@
     my $warps = '';
     foreach $exposure (@$exposures) {
-        if (($exposure->{data_state} != 'drop')&&
+        if (($exposure->{data_state} ne 'drop')&&
             (S64_IS_NOT_NULL($exposure->{magicked}))&&
             (S64_IS_NOT_NULL($exposure->{warp_id}))) {
@@ -720 +737 @@
     $command .= " -pretend " if defined $debug;
     $command .= " -dbname $dbname " if defined $dbname;
-    $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter ";
+    $command .= " -definebyquery -select_label ${label}% -select_skycell_id ${proj_cell}.% -select_filter $filter ";
     $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group ";
     $command .= " -min_num 2 -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} ";
@@ -897 +914 @@
 
     my @clean_modes = (
-        'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@',
-        'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@',
-        'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@',
+        'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@',
+        'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@',
+        'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@',
         'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage chip -stage_id @CHIP_ID@',
         'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage warp -stage_id @WARP_ID@',
@@ -910 +927 @@
             my $command = $clean_mode;
             $command =~ s/\@DBNAME\@/$dbname/g;
-            $command =~ s/\@CHIP_ID\@/$exposure->{chip_id}/;
-            $command =~ s/\@WARP_ID\@/$exposure->{warp_id}/;
-            $command =~ s/\@DIFF_ID\@/$exposure->{diff_id}/;
+            if ($command =~ /\@CHIP_ID\@/) {
+                if (S64_IS_NOT_NULL($exposure->{chip_id})) {
+                    $command =~ s/\@CHIP_ID\@/$exposure->{chip_id}/;
+                }
+                else {
+                    next;
+                }
+            }
+            if ($command =~ /\@WARP_ID\@/) {
+                if (S64_IS_NOT_NULL($exposure->{warp_id})) {
+                    $command =~ s/\@WARP_ID\@/$exposure->{warp_id}/;
+                }
+                else {
+                    next;
+                }
+            }
+            if ($command =~ /\@DIFF_ID\@/) {
+                if (S64_IS_NOT_NULL($exposure->{diff_id})) {
+                    $command =~ s/\@DIFF_ID\@/$exposure->{diff_id}/;
+                }
+                else {
+                    next;
+                }
+            }
             $command =~ s/\@LABEL\@/$exposure->{label}/;
             
@@ -925 +963 @@
         update_this_exposure($exposure);
     }
+    
+    # Cleanup quickstacks.
+    $command = "$laptool -stacks -lap_id $lap_id";
+    $command .= " -dbname $dbname" if defined $dbname;
+    
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform laptool -stacks: $error_code", "none", "none");
+    }
+    if (@$stdout_buf == 0) {
+        # Nothing to do.
+        return(0);
+    }
+    
+    my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata from laptool -inactiveexp", $lap_id);
+    
+
+    @clean_modes = (
+        'stacktool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -stack_id @STACK_ID@');
+    foreach my $stack (@$stacks) {
+        if (!S64_IS_NOT_NULL($stack->{quick_stack_id})) {
+            next;
+        }
+        foreach my $clean_mode (@clean_modes) {
+            my $command = $clean_mode;
+            $command =~ s/\@DBNAME\@/$dbname/g;
+            $command =~ s/\@LABEL\@/$stack->{label}/;
+            $command =~ s/\@STACK_ID\@/$stack->{quick_stack_id}/;
+
+            ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform cleantool: $command : $error_code", "none", "none");
+            }
+        }
+    }    
+
+    $command = "$laptool -updaterun -lap_id $lap_id";
+    $command .= " -dbname $dbname " if defined $dbname;
+    $command .= " -set_state done ";
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id);
+    }
+
 }