Changeset 30575 for trunk/ippScripts/scripts/nightly_science.pl

Timestamp:

Feb 11, 2011, 2:15:44 PM (15 years ago)

Author:

watersc1

Message:

Addition of multi-date diffs when reaching FINISHED_DIFFS.

File:

• trunk/ippScripts/scripts/nightly_science.pl (modified) (3 diffs)

trunk/ippScripts/scripts/nightly_science.pl

@@ -1266 +1266 @@
 }
 
+sub multi_date_verify_uniqueness_diff {
+    my $warp_id_1 = shift;
+    my $warp_id_2 = shift;
+    my $date = shift;
+    my $target = shift;
+
+    my $db = init_gpc_db();
+    $date =~ s/-//g;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $count = 0;
+    
+    my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
+    my $data_ref = $db->selectall_arrayref( $sth );
+    $count += $#{ $data_ref } + 1;
+
+    $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_2 AND warp2 = $warp_id_1";
+    $data_ref = $db->selectall_arrayref( $sth );
+    $count += $#{ $data_ref } + 1;
+
+    return($count);
+}
+
 sub pre_diff_queue {
     my $date = shift;
@@ -1375 +1398 @@
         $metadata_out{nsDiffState} = 'DIFFING';
     }
-
+    if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') {
+        foreach my $target (sort (keys %science_config)) {
+            if ($science_config{$target}{DIFFABLE} == 1) {
+                foreach my $filter (@filter_list) {
+                    multi_date_diff_queue($date,$target,$filter,$pretend);
+                }
+            }
+        }
+    }
 #     if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) {
 #       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
@@ -1511 +1542 @@
 
 }
+
+sub multi_date_diff_queue {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+    my $pretend = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $db = init_gpc_db();
+
+    my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+    $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
+
+    my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+    my $Npotential = 0;
+    my $Nqueued = 0;
+    
+    foreach my $object_row (@{ $object_ref }) {
+        my $this_object = shift @{ $object_row };
+#       my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+#       $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+#       $input_sth .= " ORDER BY dateobs ";
+        
+        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM ";
+        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
+        $input_sth .=   " WHERE warpRun.label = '$label' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+        $input_sth .=   " ORDER BY dateobs ";
+
+        my $warps = $db->selectall_arrayref( $input_sth );
+
+        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
+        my %comment_hash = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            $comment_hash{$this_comment} = $this_exp_id;
+        }
+        
+        if (($#{ $warps } + 1) % 2 != 0) {
+            print STDERR "md_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
+            if ($#{ $warps} + 1 == 1) {
+                print STDERR ": I can do no diffs with only one exposure.\n";
+                next;
+            }
+            else {
+                print STDERR ": I should declare an exposure to be faulty.\n";
+                my @keep_warps = ();
+#               print "@{ $warps }\n";
+                foreach my $this_warp (@{ $warps }) {
+                    my $this_comment = ${ $this_warp }[3];
+                    my $this_exp_id  = ${ $this_warp }[0];
+                    if ($comment_hash{$this_comment} == $this_exp_id) {
+                        push @keep_warps, $this_warp;
+                    }
+                    else {
+                        print STDERR "md_diff_queue: excluding $this_exp_id for $this_object\n";
+                    }
+                }
+                @{ $warps } = @keep_warps;
+#               print "@{ $warps }\n";
+            }
+        }
+        
+        while ($#{ $warps } > -1) {
+            my $input_warp = shift @{ $warps };
+            my $input_exp_id = ${ $input_warp }[0];
+            my $input_comment = ${ $input_warp }[3];
+
+            
+            my $template_warp = shift @{ $warps };
+
+            my $template_exp_id = ${ $template_warp }[0];
+            
+            my $input_warp_id = ${ $input_warp }[1];
+            my $template_warp_id = ${ $template_warp }[1];
+
+            my $input_warp_state = ${ $input_warp }[4];
+            my $template_warp_state = ${ $template_warp }[4];
+            
+            unless(defined($template_warp)&& defined($template_exp_id)) {
+                print STDERR "md_diff received an undef! $input_exp_id $input_comment $this_object T: $template_warp V: @$template_warp\n";
+                next;
+            }
+            $Npotential++;
+            
+            unless (defined($input_warp_id) && defined($template_warp_id) &&
+                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
+                print STDERR "md_Diff for this $date $target $input_exp_id ($input_warp_id $input_warp_state) $template_exp_id ($template_warp_id $template_warp_state) not fully processed\n";
+                next;
+            }
+
+            if (multi_date_verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
+                $Nqueued++;
+                print STDERR "md_Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
+                next;
+            }
+
+            my $new_data_group = "${data_group}.multi";
+            my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
+            $cmd .= "-input_label $label  -template_label $label ";
+            $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
+            $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $new_data_group ";
+            $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
+#               $cmd .= " -pretend ";
+            if (defined($pretend)) {
+                $cmd .= ' -pretend ';
+            }
+            if ($debug == 1) {
+                $cmd .= ' -pretend ';
+                print STDERR "md_Diffs would like to queue for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
+                print STDERR "md_diff_queue: $cmd\n";
+                print STDERR " $input_warp_id $template_warp_id\n";
+            }
+            
+            if (($debug == 0)&&(!defined($pretend))) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run ( command => $cmd, verbose => $verbose );
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+                }
+                $Nqueued++;
+            }
+        }
+    }
+    $metadata_out{nsDiffPotential} += $Npotential;
+    $metadata_out{nsDiffQueued}    += $Nqueued;
+#      if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
+#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+#      }        
+
+}