Changeset 30120 for trunk/ippScripts/scripts/nightly_science.pl

Timestamp:

Dec 20, 2010, 5:52:38 PM (15 years ago)

Author:

watersc1

Message:

updates to nightly science code to handle register failures and stacks and diffs

File:

• trunk/ippScripts/scripts/nightly_science.pl (modified) (11 diffs)

trunk/ippScripts/scripts/nightly_science.pl

@@ -396 +396 @@
     exit(0);
 }
-exit(10);
+
 if (defined($check_confirm_stacks) || defined($test_mode)) {
     $metadata_out{nsStackState} = 'CONFIRM_STACKING';
@@ -409 +409 @@
     exit(0);
 }
-
+exit(10);
 if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsSSState} = 'CHECKSWEETSPOT';
@@ -1097 +1097 @@
     my $pretend = shift;
 
+    my $Npotential = 0;
+    my $Nqueued = 0;
     foreach my $target (sort (keys %science_config)) {
         if ($science_config{$target}{STACKABLE} == 1) {
@@ -1120 +1122 @@
                     next;
                 }
+                $Npotential++;
                 if ($Nalready != 0) {
+                    $Nqueued++;
                     if ($debug == 1) {
                         print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
@@ -1137 +1141 @@
                     }
                     stack_queue($date,$target,$filter);
+                    $Nqueued ++;
                 }
                 if (defined($pretend)) {
@@ -1152 +1157 @@
         }
     }
+    $metadata_out{nsStackPotential} = $Npotential;
+    $metadata_out{nsStackQueued}    = $Nqueued;
+    if (($Npotential == $Nqueued)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
+        $metadata_out{nsStackState} = 'FINISHED_STACKS';
+    }   
 }
 
@@ -1231 +1241 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
 
-    my $sth = "SELECT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
+    my $count = 0;
+    
+    my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
     my $data_ref = $db->selectall_arrayref( $sth );
-
-    return($#{ $data_ref } + 1);
+    $count += $#{ $data_ref } + 1;
+
+    $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_2 AND warp2 = $warp_id_1";
+    $data_ref = $db->selectall_arrayref( $sth );
+    $count += $#{ $data_ref } + 1;
+
+    return($count);
 }
 
@@ -1276 +1293 @@
             foreach my $filter (@filter_list) {
                 my ($NprocChips,$NprocWarps) = pre_diff_queue($date,$observing_state,$target,$filter);
-                if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) {
-                    if ($debug == 1) {
-                        print STDERR "execute_diffs: Target $target in filter $filter on $date is not fully processed. ($NprocChips $NprocWarps)\n";
-                    }
-                    $metadata_out{nsDiffState} = 'FORCETOWARP';
-                    next;
-                }
+#               if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) {
+#                   if ($debug == 1) {
+#                       print STDERR "execute_diffs: Target $target in filter $filter on $date is not fully processed. ($NprocChips $NprocWarps)\n";
+#                   }
+#                     $metadata_out{nsDiffState} = 'FORCETOWARP';
+#                     next;
+#                 }
                 if ($NprocChips == 0) {
                     if ($debug == 1) {
@@ -1326 +1343 @@
 
     my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+    my $Npotential = 0;
+    my $Nqueued = 0;
     
     foreach my $object_row (@{ $object_ref }) {
         my $this_object = shift @{ $object_row };
-        my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
         $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
         $input_sth .= " ORDER BY dateobs ";
         
         my $warps = $db->selectall_arrayref( $input_sth );
+
+        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
+        my %comment_hash = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            $comment_hash{$this_comment} = $this_exp_id;
+        }
         
         if (($#{ $warps } + 1) % 2 != 0) {
-            print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps }\n";
-            next;
+            print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
+            if ($#{ $warps} + 1 == 1) {
+                print STDERR ": I can do no diffs with only one exposure.\n";
+                next;
+            }
+            else {
+                print STDERR ": I should declare an exposure to be faulty.\n";
+                my @keep_warps = ();
+#               print "@{ $warps }\n";
+                foreach my $this_warp (@{ $warps }) {
+                    my $this_comment = ${ $this_warp }[3];
+                    my $this_exp_id  = ${ $this_warp }[0];
+                    if ($comment_hash{$this_comment} == $this_exp_id) {
+                        push @keep_warps, $this_warp;
+                    }
+                    else {
+                        print STDERR "diff_queue: excluding $this_exp_id for $this_object\n";
+                    }
+                }
+                @{ $warps } = @keep_warps;
+#               print "@{ $warps }\n";
+            }
         }
         
         while ($#{ $warps } > -1) {
             my $input_warp = shift @{ $warps };
+            my $input_exp_id = ${ $input_warp }[0];
+            my $input_comment = ${ $input_warp }[3];
+
+            
             my $template_warp = shift @{ $warps };
-            my $input_exp_id = ${ $input_warp }[0];
+
             my $template_exp_id = ${ $template_warp }[0];
             
             my $input_warp_id = ${ $input_warp }[1];
             my $template_warp_id = ${ $template_warp }[1];
-            
+
+            $Npotential++;
             if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
+                $Nqueued++;
                 print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id\n";
                 next;
@@ -1366 +1420 @@
                 $cmd .= ' -pretend ';
                 print STDERR "diff_queue: $cmd\n";
+                print STDERR " $input_warp_id $template_warp_id\n";
             }
             
@@ -1378 +1433 @@
         }
     }
+    $metadata_out{nsDiffPotential} += $Npotential;
+    $metadata_out{nsDiffQueued}    += $Nqueued;
+    if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
+        $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+    }   
+
 }