Changeset 32630 for branches/eam_branches/ipp-20110906/ippScripts/scripts/lap_science.pl

Timestamp:

Nov 8, 2011, 2:44:12 PM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20110906

Files:

• . (modified) (1 prop)
• ippScripts/scripts/lap_science.pl (modified) (22 diffs)

branches/eam_branches/ipp-20110906/ippScripts/scripts/lap_science.pl

@@ -27 +27 @@
 my ( $camera, $dbname);
 my ( $lap_id );
+my ( $queue_list );
 my ( $chip_mode, $monitor_mode, $cleanup_mode);
 
@@ -44 +45 @@
     
     'lap_id=s'     => \$lap_id,
+    'queue_list=s' => \$queue_list,
 
     'chip_mode'    => \$chip_mode,
@@ -200 +202 @@
     return($exposure);
 }
+
+sub remake_this_exposure_by_update {
+    my $exposure = shift;
+
+    my @utctime = gmtime();
+    $utctime[5] += 1900;
+    $utctime[4] += 1;
+
+    my $label = $exposure->{label};
+
+    my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
+    my $data_group = "${label}.${date}";
+
+    my $chiptool_info_cmd = "chiptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} ";
+    $chiptool_info_cmd   .= " -dbname $dbname " if defined $dbname;
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $chiptool_info_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform chiptool -listrun: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    my $chips = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata from chiptool -listrun", $exposure->{exp_id}, $data_group);
+    # There should be only one.
+    my $chip = ${ $chips }[0];
+    my $chip_magicDS_id = 0;
+    if ($chip) {
+        $chip_magicDS_id = $chip->{magic_ds_id};
+    }
+    if ($chip_magicDS_id == 0) {
+#       return(&remake_this_exposure($exposure));
+    }
+
+    my $warptool_info_cmd = "warptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} ";
+    $warptool_info_cmd   .= " -dbname $dbname " if defined $dbname;
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $warptool_info_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform warptool -listrun: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    my $warps = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata from warptool -listrun", $exposure->{exp_id}, $data_group);
+    # There should be only one.
+    my $warp = ${ $warps }[0];
+    my $warp_id = 0;
+    if ($warp) {
+        $warp_id = $warp->{warp_id};
+    }
+    else {
+        $exposure->{data_state} = 'drop';
+        return($exposure);
+    }   
+    if ($warp_id == 0) {
+        # Handle this correctly. Probably want to drop the exposure here.
+        $exposure->{data_state} = 'drop';
+        return($exposure);
+    }
+
+    if (($chip->{state} eq 'goto_cleaned')||
+        ($warp->{state} eq 'goto_cleaned')||
+        ($chip->{dsRun_state} eq 'goto_cleaned')) {
+        $exposure->{data_state} = 'pending_update';
+        &update_this_exposure($exposure);
+        return($exposure);
+    }
+
+    
+    my $chiptool_update_cmd = "chiptool -setimfiletoupdate -chip_id $exposure->{chip_id} -set_label $label";
+    $chiptool_update_cmd   .= " -dbname $dbname " if defined $dbname;
+    my $magicDS_update_cmd  = "magicdstool -setfiletoupdate -magic_ds_id $chip_magicDS_id -set_label $label";
+    $magicDS_update_cmd    .= " -dbname $dbname " if defined $dbname;
+    my $warptool_update_cmd = "warptool -setskyfiletoupdate -warp_id $warp_id -set_label $label";
+    $warptool_update_cmd   .= " -dbname $dbname " if defined $dbname;
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $chiptool_update_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    if ($chip_magicDS_id != 0) {    
+        ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $magicDS_update_cmd, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
+        }
+    }
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $warptool_update_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform warptool -setskyfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    
+    $exposure->{active} = 1;
+    return($exposure);
+}
+        
+sub determine_if_can_update {
+    my $exposure = shift;
+    
+    if (S64_IS_NOT_NULL($exposure->{chip_id})) {
+        if (($exposure->{chip_state} eq 'cleaned')||
+            ($exposure->{chip_state} eq 'goto_cleaned')||
+            ($exposure->{chip_state} eq 'error_cleaned')) {
+            return(1);
+        }
+    }
+
+    return(0);
+}
+        
 
 # This is the "user level" subroutine.
@@ -240 +360 @@
         my $comment = $exposure->{comment};
 
-        # This is a hack to fix old exposures that have no object.
+        my $updateable = determine_if_can_update($exposure);
+
+        # This is a hack to fix old exposures that have no usable object/comment data.
         unless(defined($comment)) {
             $comment = '';
@@ -247 +369 @@
             if ($comment =~ /3pi_/) {
                 $object = $comment;
-                $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/;
+                $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/F1 $1/;
             }
             elsif ($comment =~ / ps1_/) {
                 $object = $comment;
-                $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/;
-            }
-            else {
-                $exposure->{data_state} = 'new';
+                $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/F2 $1/;
+            }
+            elsif ($comment =~ /ThreePi.*3pi_/) {
+                $object = $comment;
+                $object =~ s/^ThreePi . (\S+? 3pi_\d\d_\d\d\d\d) .*$/F3 $1/;
+            }           
+            elsif ($comment =~ /ThreePi.*ps1_/) {
+                $object = $comment;
+                $object =~ s/^ThreePi . (\S+? ps1_\d\d_\d\d\d\d) .*$/F4 $1/;
+            }           
+            elsif ($comment =~ /ThreePi /) {
+                $object = $comment;
+                $object =~ s/^ThreePi . (\S+? \d\d\d\d) .*$/F5 $1/;
+            }           
+            elsif (($comment =~ /focus/i)||
+                   ($comment =~ /test/i)||
+                   ($comment =~ /hyster/i)||
+                   ($comment =~ /dither/i)||
+                   ($comment =~ /camera/i)
+                ){
+                # This is junk that shouldn't exist.
+                $object = 'DROP';
+                $exposure->{data_state} = 'drop';
                 $exposure->{pairwise} = 0;
-                $exposure->{private} = 1;
-                $exposure->{pair_id} = 9223372036854775807;
-                update_this_exposure($exposure);
-                $exposure = remake_this_exposure($exposure); 
-                $counter++;
+                $exposure->{pair_id} =  9223372036854775807;
+                $exposure->{active} = 0;
+                &update_this_exposure($exposure);
+                $counter++; # To ensure everyone else is consistent
                 next;
             }
         }
 
+        # Determine the current state of chipRuns for these exposures, and update/remake as needed.
         if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id
             if (($pairwise) && !($pair_id)) {
@@ -270 +411 @@
                 &my_die("Exposure $exp_id for $lap_id is declared pairwise without a defined pair", $lap_id);
             }
+            $exposure->{data_state} = 'exists';
+            if ($updateable) { # We know about this run, but the data needs to be regenerated.
+                $exposure->{data_state} = 'update';
+                $exposure = remake_this_exposure_by_update($exposure);
+            }
+            &update_this_exposure($exposure);
             $matching{$object}{$comment} = $exp_id;
             $indexing{$exp_id} = $counter;
             $counter++; 
-            next;
         }
         else { # We do not already have a chip_id.  
             # Make a chipRun, and update the exposure.
+            $exposure->{data_state} = 'run';
             $exposure = remake_this_exposure($exposure); 
-            
             # Save our information for diff pairing.
+            &update_this_exposure($exposure);
             $matching{$object}{$comment} = $exp_id;
             $indexing{$exp_id} = $counter;
             $counter++;
         }
+
         if ($verbose) {
             print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n";
@@ -296 +444 @@
             print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n";
         }
-        @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff;
+        @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; # This is effectively a sort by dateobs.
         
         if (( $#exp_ids_to_diff + 1) % 2 != 0) { # We have an odd number of exposures, even after comment filtering
@@ -309 +457 @@
             my $exp_B = ${ $exposures }[$indexing{$exp_id_B}];
             print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n";
-            $exp_A->{pairwise} = 1;
-            $exp_A->{private} = 0;
-            $exp_A->{pair_id} = $exp_B->{chip_id};
-            
-            $exp_B->{pairwise} = 1;
-            $exp_B->{private} = 0;
-            $exp_B->{pair_id} = $exp_A->{chip_id};
-            
-            if ($verbose) {
-                print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n";
+            if ($exp_A->{diff_id} == $exp_B->{diff_id}) {
+                $exp_A->{pairwise} = 1;
+                $exp_A->{pair_id} = $exp_B->{chip_id};
+                
+                $exp_B->{pairwise} = 1;
+                $exp_B->{pair_id} = $exp_A->{chip_id};
+                if ($verbose) {
+                    print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n";
+                }
+            }
+            else {
+                $exp_A->{pairwise} = 0;
+                $exp_B->{pairwise} = 0;
             }
         }
@@ -331 +482 @@
             $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error.
         }
-        if (!($exposure->{pairwise})) {
-            $exposure->{private} = 1; # If this isn't being pairwise diffed, it needs to be private
-        }
-
+        
         update_this_exposure($exposure);
     }
@@ -457 +605 @@
     # Things I want to know before I'm through
 #     my $needs_qstack = 0;
-    my $needs_something_remade = 0;
-    my $needs_something_private = 0;
+    my $needs_something_remade = 0;  # I never need something remade, because we're committed to one-exp=one-chip
+    my $needs_something_private = 0; # I never need something private, as that is calculated by the difference engine.
 #     my $can_qstack = 0;
      my $have_diff = 0;
@@ -480 +628 @@
             else { # We claimed to be pairwise, but do not have a valid pair_id.
                 $exposure->{pairwise} = 0;
-                $exposure->{private} = 1;
                 &update_this_exposure($exposure);
             }
@@ -487 +634 @@
         if  ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through.
             $are_warped++;
-#           $can_qstack ++;
             $can_diff ++;
             $have_diff ++;
             $are_magicked ++;
-#           $can_fstack ++;         
+            next;
+        }
+
+        if ($exposure->{data_state} eq 'pending_update') {
+            $exposure->{data_state} = 'update';
+            $exposure = &remake_this_exposure_by_update($exposure);
+            &update_this_exposure($exposure);
             next;
         }
         
         if ($exposure->{private}) { # I've declared this exposure private to this lapRun.
-#           $needs_qstack++;
             push @lonely_exposures, $exposure;
-#           $lonely_exposures++;
         }
         
-        if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun
-            $needs_something_remade = 1;
-            $exposure = remake_this_exposure($exposure);
-        }
+#       if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun
+#           $needs_something_remade = 1;
+#           $exposure = remake_this_exposure($exposure);
+#       }
         # Do quality checks here
         my $is_bad_quality = 0;
@@ -536 +686 @@
                     ($exposure->{diffRun_state} eq 'full')) {
                 $are_warped++;
-#               $needs_qstack++;
-                $needs_something_private = 1;
                 if ($companion) {
-                    $companion->{private} = 1;
                     $companion->{pairwise} = 0;
                     &update_this_exposure($companion);
-                    push @lonely_exposures, $companion;
-#                   $lonely_exposures++;
-                }
-                $exposure->{private} = 1;
+#                   push @lonely_exposures, $companion;
+                }
                 $exposure->{pairwise} = 0;
             }
             $exposure->{data_state} = 'drop';
             &update_this_exposure($exposure);
+            next;
         }
         
@@ -558 +704 @@
 #           $can_qstack ++;
             $can_diff ++;
-        }
-        if (($exposure->{magicked}&&
-             &S64_IS_NOT_NULL($exposure->{magicked}))) {  # This exposure has been magicked, so it is through with diff.
-            $are_magicked++;
-#           $can_fstack ++;
+            $exposure->{data_state} = 'to_diff';
         }
         if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) {
             $have_diff ++;
-        }
+            $exposure->{data_state} = 'to_magic';
+        }
+        if (($exposure->{magicked})&&
+            ($exposure->{warpRun_state})&&
+            ($exposure->{warpRun_state} eq 'full')&&
+            (&S64_IS_NOT_NULL($exposure->{magicked}))) {  # This exposure has been magicked, so it is through with diff.
+            $are_magicked++;
+            $exposure->{data_state} = 'full';
+#           $can_fstack ++;
+        }
+        &update_this_exposure($exposure);
     }
 
@@ -575 +727 @@
     print "STATUS: HAVE_FSTACK:     $have_fstack\n";
 
-#    print "STATUS: NEEDS_QSTACK:    $needs_qstack\n";
     print "STATUS: NEEDS_REMADE:    $needs_something_remade\n";
     print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n";
     print "STATUS: ARE WARPED:      $are_warped\n";
     print "STATUS: ARE MAGICKED:    $are_magicked\n";
-#    print "STATUS: CAN_QSTACK:      $can_qstack\n";
     print "STATUS: CAN_DIFF:        $can_diff\n";
     print "STATUS: HAVE_DIFF:       $have_diff\n";
-#    print "STATUS: CAN_FSTACK:      $can_fstack\n";
 
     print "STATUS: TOTAL_EXPOSURES: $total_exposures\n";
@@ -924 +1073 @@
             next;
         }
-        if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # Not sure how this would happen, but still. ## This happens when we inherit a complete exposure.
+        if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # This happens when we inherit a complete exposure.
             next;
         }
@@ -964 +1113 @@
         }
         else { # warp-qstack
+            # We need to decide if we can make a warp-stack diff:
+            if (&can_warp_stack_diff_be_made($exposure)) {
 #           next; 
 #           $command .= '-pretend';
-            $command .= " -definewarpstack -available -good_frac 0.2 ";
-            $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} ";
-            $already_queued{$exposure->{warp_id}} = 1;
+                $command .= " -definewarpstack -available -good_frac 0.2 ";
+                $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} ";
+                $already_queued{$exposure->{warp_id}} = 1;
+                $exposure->{private} = 0;
+                &update_this_exposure($exposure);
+            }
+            else {
+                $exposure->{private} = 1;
+                $exposure->{data_state} = 'drop';
+                $already_queued{$exposure->{warp_id}} = 1;
+                &update_this_exposure($exposure);
+                next;
+            }
         }
         
@@ -1006 +1167 @@
 }
 
+sub can_warp_stack_diff_be_made {
+    my $exposure = shift;
+    my $lap_id   = $exposure->{lap_id};
+    my $warp_id  = $exposure->{warp_id};
+
+    my $command = "$laptool -diffcheck -lap_id $lap_id -warp_id $warp_id";
+    $command .= " -dbname $dbname " if defined $dbname;
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform laptool -diffcheck: $error_code", $lap_id, $warp_id);
+    }
+    if (@$stdout_buf == 0) {
+        return(0);
+    }
+    my $skycells = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata from laptool -diffcheck", $lap_id, $warp_id);
+    foreach my $skycell (@$skycells) {
+        unless (defined($skycell->{stack_state})) {
+            return(0);
+        }
+        if ($skycell->{stack_state} ne 'full') {
+            return(0);
+        }
+        if ($skycell->{warp_state} ne 'full') {
+            return(0);
+        }
+        unless (S64_IS_NOT_NULL($skycell->{stack_id})) {
+            return(0);
+        }
+    }
+    return(1);
+}
+
+
 # Deactivate all exposures in this run.
 sub deactivate_exposures {
     my $exposures = shift;
     foreach my $exposure (@$exposures) {
-        $exposure->{active} = 0;
-        update_this_exposure($exposure);
+        if ($exposure->{private} == 1) {  # This can probably be relaxed since we can update an undestreaked warp.
+            $exposure->{active} = 1;
+        }
+        else {
+            $exposure->{active} = 0;
+        }
+        update_this_exposure($exposure);        
     }
 }
@@ -1022 +1225 @@
 sub cleanup_mode {
     my $lap_id = shift;
+
+    if (defined($queue_list)) {
+        my $successful = 1;
+        open(Q,$queue_list) or ($successful = 0);
+        if ($successful == 1) {
+            while(<Q>) {
+                chomp;
+                my $cmd = $_;
+                my ($projection_cell,$seq_id,$filter,$label);
+                $projection_cell = $cmd;
+                $projection_cell =~ s/.*?-projection_cell (skycell.\w+?) .*/$1/;
+                $seq_id = $cmd;
+                $seq_id =~ s/.*-seq_id (\d+?) .*/$1/;
+                $filter = $cmd;
+                $filter =~ s/.*-filter (\w\.00000) .*/$1/;
+                if ($filter =~ /\s+?/) {
+                    $filter =~ s/.*-filter (\w\.00000)$/$1/;
+                }
+                $label = $cmd;
+                $label =~ s/.*-label (.+?) .*/$1/;
+                
+                my $response;
+                chomp($response = 
+                      `laptool -dbname gpc1 -listrun -projection_cell $projection_cell -seq_id $seq_id -filter $filter -label $label -simple`);
+                my $state = (split /\s+/, $response)[5];
+                
+                unless (defined($state)) {
+                    if ($verbose) {
+                        print "Queuing: $cmd\n";
+                    }
+                    system($cmd);
+                    
+                    my $i = 0;
+                    do {
+                        sleep(5);
+                        chomp($response = 
+                              `laptool -dbname gpc1 -listrun -projection_cell $projection_cell -seq_id $seq_id -filter $filter -label $label -simple`);
+                        $state = (split /\s+/, $response)[5];
+                        $i++;
+                        unless(defined($state)) {
+                            $successful = 0;
+                        }
+                    } while (($state ne 'run')&&($i < 20)&&($successful));
+                    last;
+                }
+            }
+            close(Q);
+        }
+    }
+
     my $command = "$laptool -inactiveexp -lap_id $lap_id ";
     $command .= " -dbname $dbname " if defined $dbname;
@@ -1043 +1296 @@
         'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@',
         'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@',
+        'magictool -dbname @DBNAME@ -updaterun -set_state full -set_label @LABEL@.old -magic_id @MAGIC_ID@',
         'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage chip -stage_id @CHIP_ID@',
         'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage warp -stage_id @WARP_ID@',
         'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage diff -stage_id @DIFF_ID@');
+    
     foreach my $exposure (@$exposures) {
         if ($exposure->{is_in_use}) {
@@ -1072 +1327 @@
                 if (S64_IS_NOT_NULL($exposure->{diff_id})) {
                     $command =~ s/\@DIFF_ID\@/$exposure->{diff_id}/;
+                }
+                else {
+                    next;
+                }
+            }
+            if ($command =~ /\@MAGIC_ID\@/) {
+                if ((S64_IS_NOT_NULL($exposure->{magicked}))&&
+                     ($exposure->{magicked} > 0)) {
+                    $command =~ s/\@MAGIC_ID\@/$exposure->{magicked}/;
                 }
                 else {
@@ -1139 +1403 @@
         &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id);
     }
-
+    return(0);
 }