Changeset 28484 for branches/pap/ippScripts/scripts/automate_stacks.pl

Timestamp:

Jun 24, 2010, 2:59:09 PM (16 years ago)

Author:

Paul Price

Message:

Merging trunk in advance of reintegrating into trunk.

Location:

branches/pap

Files:

• . (modified) (1 prop)
• ippScripts/scripts/automate_stacks.pl (modified) (24 diffs)

branches/pap/ippScripts/scripts/automate_stacks.pl

@@ -17 +17 @@
 my $missing_tools = 0;
 my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
+my $dqstatstool = can_run('dqstatstool') or (warn "Can't find dqstatstool" and $missing_tools = 1);
 my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
 my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1);
@@ -27 +29 @@
     exit($PS_EXIT_CONFIG_ERROR);
 }
-
-# my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000');
-# my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10',
-#                  'STS','SVS','SweetSpot','ThreePi');
-# my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02',
-#                  'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06',
-#                  'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10',
-#                  'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0');
-# my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%',
-#                   'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%',
-#                   'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%',
-#                   'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%');
-# my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1,
-#                   'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1,
-#                   'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1,
-#                   'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0);
-# my $retention_time = 9000;  # days.
-
 
 my $db;
@@ -59 +43 @@
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
 my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
+my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats);
 
 GetOptions(
@@ -81 +66 @@
     'check_stacks'         => \$check_stacks,
     'queue_stacks'         => \$queue_stacks,
+    'check_detrends'       => \$check_detrends,
+    'queue_detrends'       => \$queue_detrends,
+    'check_dqstats'        => \$check_dqstats,
+    'queue_dqstats'        => \$queue_dqstats,
     'check_diffs'          => \$check_diffs,
     'queue_diffs'          => \$queue_diffs,
@@ -105 +94 @@
            --check_stacks         Confirm that stacks can be built.
            --queue_stacks         Issue stacktool commands to queue stacks.
+           --check_detrends       Confirm that detrend verify runs can be built.
+           --queue_detrends       Issue dettool commands to queue detrend verify runs.
+           --check_dqstats        Confirm that dqstats tables can be built.
+           --queue_dqstats        Issue dqstatstool commands to queue dqstat tables.
            --check_diffs          Confirm that diffs can be done.
            --queue_diffs          Issue difftool commands to queue diffs.
@@ -115 +108 @@
           ) unless
     defined $check_registration or defined $define_burntool or defined $queue_burntool or
-    defined $queue_chips or defined $queue_stacks or
-    defined $check_chips or defined $check_stacks or
+    defined $queue_chips or defined $queue_stacks or $queue_detrends or $queue_dqstats or
+    defined $check_chips or defined $check_stacks or $check_detrends or $check_dqstats or
     defined $test_mode or defined $clean_old or defined $check_mode;
 
@@ -126 +119 @@
 my %object_list = ();
 my %comment_list= ();
+my %cleanmods_list = ();
 my %stackable_list = ();
 my %reduction_class = ();
+my @detrend_list = ();
+my %dettype_list = ();
+my %exptype_list = ();
+my %refID_list = ();
+my %refIter_list = ();
+my %detfilter_list = ();
+my %detmax_list = ();
 my %clean_commands = ();
 my %clean_retention = ();
 my %noclean_list = ();
+my %clean_alternate = ();
 # Grab the configuration data.
 my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
@@ -137 +139 @@
 unless ($success) {
     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR);
+    &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR);
 }
 
@@ -156 +158 @@
                 $clean_retention{$this_mode} = ${ $mentry }{value};
             }
+            elsif (${ $mentry }{name} eq 'ALTERNATE_CMD') {
+                $clean_alternate{$this_mode} = ${ $mentry }{value};
+            }
         }
     }
-#    if (${ $entry }{name} eq 'RETENTION_TIME') {
-#        $retention_time = ${ $entry }{value};
-#    }
     elsif (${ $entry }{name} eq 'FILTERS') {
         push @filter_list, ${ $entry }{value};
@@ -193 +195 @@
                 $noclean_list{$this_target} = ${ $tentry }{value};
             }
-        }
-    }
-}
-
-
-
+            else {
+                if (exists($clean_commands{ ${ $tentry }{name} })) {
+                    $cleanmods_list{$this_target}{${ $tentry }{name} } = ${ $tentry }{value};
+                }
+            }
+        }
+    }
+    elsif (${ $entry }{name} eq 'DETRENDS') {
+        my @detrend_data = @{ ${ $entry }{value} };
+        my $this_detrend = '';
+        foreach my $dentry (@detrend_data) {
+            if (${ $dentry }{name} eq 'NAME') {
+                $this_detrend = ${ $dentry }{value};
+                push @detrend_list, $this_detrend;
+            }
+            elsif (${ $dentry }{name} eq 'DETTYPE') {
+                $dettype_list{$this_detrend} = ${ $dentry }{value};
+            }
+            elsif (${ $dentry }{name} eq 'EXPTYPE') {
+                $exptype_list{$this_detrend} = ${ $dentry }{value};
+            }
+            elsif (${ $dentry }{name} eq 'REF_ID') {
+                $refID_list{$this_detrend} = ${ $dentry }{value};
+            }
+            elsif (${ $dentry }{name} eq 'REF_ITER') {
+                $refIter_list{$this_detrend} = ${ $dentry }{value};
+            }
+            elsif (${ $dentry }{name} eq 'FILTER') {
+                $detfilter_list{$this_detrend} = ${ $dentry }{value};
+            }
+            elsif (${ $dentry }{name} eq 'MAX') {
+                $detmax_list{$this_detrend} = ${ $dentry }{value};
+            }
+        }
+    }           
+}
 
 unless(defined($date)) {
@@ -257 +289 @@
     unless (defined($test_mode) || defined($check_mode)) { exit(0); }
 }
+if (defined($check_dqstats) || defined($test_mode)) {
+    $metadata_out{nsState} = 'CHECKDQSTATS';
+    &execute_dqstats($date,"pretend");
+    return_metadata($date);
+    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
+}
+if (defined($queue_dqstats) || defined($test_mode)) {
+    $metadata_out{nsState} = 'QUEUEDQSTATS';
+    &execute_dqstats($date);
+    return_metadata($date);
+    unless (defined($test_mode)) { exit(0); }
+}
+if (defined($check_detrends) || defined($test_mode) || defined($check_mode)) {
+    $metadata_out{nsState} = 'CHECKDETRENDS';
+    &execute_detrends($date,"pretend");
+    return_metadata($date);
+    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
+}
+if (defined($queue_detrends)) {
+    $metadata_out{nsState} = 'QUEUEDETRENDS';
+    &execute_detrends($date);
+    return_metadata($date);
+    exit(0);
+}
 if (defined($define_burntool) || defined($test_mode)) {
     $metadata_out{nsState} = 'QUEUEBURNING';
@@ -347 +403 @@
         if ($summit_fault) {
             print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault";
-            if ($exp_type ne 'OBJECT') {
+            if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) {
                 print STDERR " (but I don't care).\n";
             }
@@ -357 +413 @@
         elsif (!$download_state or $download_state eq 'run') {
             print STDERR "check_summit_copy: $date $exp_name has download_state $download_state";
-            if ($exp_type ne 'OBJECT') {
+            if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) {
                 print STDERR " (but I don't care).\n";
             }
@@ -367 +423 @@
         elsif (!$new_state or $new_state eq 'run' ) {
             print STDERR "check_summit_copy: $date $exp_name has new_state $new_state";
-            if ($exp_type ne 'OBJECT') {
+            if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) {
                 print STDERR " (but I don't care).\n";
             }
@@ -498 +554 @@
 
     my $cmd = "$chiptool";
-    $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp ';
+    $cmd .= " -simple -dbname $dbname -definebyquery -set_end_stage warp ";
     $cmd .= " -set_label $label ";
     $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
@@ -612 +668 @@
 
 #
+# DQstats
+################################################################################
+
+sub construct_dqstats_cmd {
+    my $date = shift;
+
+    my $select = "-dateobs_end ${date}T23:59:59 ";
+
+    my $cmd = "$dqstatstool";
+    $cmd .= " -simple -dbname $dbname -definebyquery ";
+    $cmd .= " $select ";
+    $cmd .= " -label %.nightlyscience ";
+    $cmd .= " -set_label dqstats.nightlyscience ";
+    if ($debug == 1) {
+        $cmd .= ' -pretend ';
+    }
+    print STDERR "$cmd\n";
+    return($cmd);
+}
+
+sub pre_dqstats_queue {
+    my $date = shift;
+    
+#     my $db = init_gpc_db();
+#     my $trunc_date = $date; $trunc_date =~ s/-//g;
+
+#     my $where = " label LIKE '%.nightlyscience' AND data_group' ";
+#     my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new' OR state = 'cleaned') AND $where ";
+#     my $cam_sth = "SELECT * from camRun WHERE state = 'full' AND $where ";
+
+#     my $chip_ref = $db->selectall_arrayref( $chip_sth );
+#     my $cam_ref = $db->selectall_arrayref( $cam_sth );
+    
+    my $command = construct_dqstats_cmd($date) . ' -pretend ';
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $command, verbose => $verbose );
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dqstatstool: $error_code",1,1,,$date, $PS_EXIT_SYS_ERROR);
+    }
+    
+    my @input_exposures = split /\n/, (join '', @$stdout_buf);
+
+    return($#input_exposures + 1,1,1); # $#{ $chip_ref } + 1, $#{ $cam_ref } + 1);
+}
+ 
+sub dqstats_queue {
+    my $date = shift;
+
+    my $command = construct_dqstats_cmd($date);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $command, verbose => $verbose );
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dqstatstool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+    return(0);
+
+}
+sub execute_dqstats {
+    my $date = shift;
+    my $pretend = shift;
+    my ($Nexposures,$Nchips,$Ncams) = pre_dqstats_queue($date);
+    if ($Nexposures == 0) {
+        print STDERR "execute_dqstats: No exposures on date $date.\n";
+    }
+    elsif ($Ncams != $Nchips) {
+        print STDERR "execute_dqstats: Not done processing data through camera stage.\n";
+    }
+    else {
+        unless(defined($pretend)) {
+            dqstats_queue($date);
+        }
+    }
+}   
+#
+# Detrend verification
+################################################################################
+
+sub construct_dettool_cmd {
+    my $date = shift;
+    my $target = shift;
+
+    my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target);
+    
+    my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
+    $date =~ s/-//g;
+
+    my $cmd = "$dettool";
+#    $cmd .= " -pretend ";
+    $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type ";
+    $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter ";
+    $cmd .= " $select ";
+    $cmd .= " -inst $camera ";
+    $cmd .= " -select_exp_type $exp_type ";
+    $cmd .= " -select_filter $filter " if defined($filter);
+    $cmd .= " -workdir $workdir ";
+    $cmd .= " -label $label ";
+    if ($maxN > 0) {
+        $cmd .= " -random_subset -random_limit $maxN ";
+    }
+    if ($debug == 1) {
+        $cmd .= ' -pretend ';
+    }
+    print STDERR "$cmd\n";
+    return($cmd);
+}    
+
+sub verify_uniqueness_detverify {
+    my $date = shift;
+    my $target = shift;
+
+    my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target);
+    
+    my $db = init_gpc_db();
+    my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter";
+    my $data_ref = $db->selectall_arrayref( $sth );
+
+    return($#{ $data_ref } + 1);
+}
+
+sub pre_detrend_queue {
+    my $date = shift;
+    my $target = shift;
+    
+    my $command = construct_dettool_cmd($date,$target) . ' -pretend ';
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR);
+    }
+    
+    my @input_exposures = split /\n/, (join '', @$stdout_buf);
+    return($#input_exposures + 1);
+}
+
+sub detrend_queue {
+    my $date = shift;
+    my $target = shift;
+    
+    my $command = construct_dettool_cmd($date,$target);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+    $metadata_out{nsState} = 'DETREND_QUEUED';
+    return(0);
+
+}
+sub execute_detrends {
+    my $date = shift;
+    my $pretend = shift;
+    my $exposures = 0;
+    foreach my $target (@detrend_list) {
+        my ($Nexposures) = pre_detrend_queue($date,$target);
+        if ($Nexposures == 0) {
+            print STDERR "execute_detrends: Target $target on $date had no exposures.\n";
+            next;
+        }
+        $exposures++;
+        unless(defined($pretend)) {
+            detrend_queue($date,$target);
+        }
+    }
+    if ($exposures == 0) {
+        $metadata_out{nsState} = 'DETREND_DROP';
+    }
+    if (($metadata_out{nsState} eq 'CHECKDETRENDS') && ($exposures > 0)) {
+        $metadata_out{nsState} eq 'QUEUE_DETRENDS';
+    }
+}
+
+#
 # Stacking
 ################################################################################
@@ -627 +859 @@
     my $cmd = "$stacktool";
 #    $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 ';  # Probably silly, but I want to be safe and not overwrite
-    $cmd .= ' -simple -dbname gpc1 -definebyquery ';
+    $cmd .= " -simple -dbname $dbname -definebyquery ";
     $cmd .= " -set_label $label -select_label $label ";
     $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
@@ -768 +1000 @@
 
     my $command = $clean_commands{$mode};
-    my $retention_time = $clean_retention{$mode};
+    my $retention_time;
+    if (exists($cleanmods_list{$target}{$mode})) {
+        $retention_time = $cleanmods_list{$target}{$mode};
+    }
+    else {
+        $retention_time = $clean_retention{$mode};
+    }
+    if ($retention_time <= 0) {
+        return("no clean","true");
+    }
 
     my ($year,$month,$day) = split /-/,$date;
@@ -774 +1015 @@
                                hour => 0, minute => 0, second => 0, nanosecond => 0,
                                time_zone => 'Pacific/Honolulu');
+        
     $dt->subtract(days => $retention_time);
     my $cleaning_date = $dt->ymd;
@@ -779 +1021 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);
     my $args = $command;
-    $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
+    if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) {
+        $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -time_stamp_end $cleaning_date ";
+    }
+    else {
+        $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
+    }
     if ($debug == 1) {
         $args .= ' -pretend ';
@@ -790 +1037 @@
     my $pretend = shift;
 
-    foreach my $mode (keys (%clean_commands)) {
-        foreach my $target (@target_list) {
-            if (exists($noclean_list{$target})) {
+    foreach my $mode (sort (keys (%clean_commands))) {
+        if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) {
+            my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode);
+            if ($cleaning_date eq 'no clean') {
                 next;
             }
-            my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
-
             print STDERR "$command\n";
             if (!(defined($pretend) || $debug == 1)) {
@@ -808 +1054 @@
             }
         }
+        else {
+            foreach my $target (@target_list) {
+                if (exists($noclean_list{$target})) {
+                    next;
+                }
+                my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
+                if ($cleaning_date eq 'no clean') {
+                    next;
+                }               
+                print STDERR "$command\n";
+                if (!(defined($pretend) || $debug == 1)) {
+#           print STDERR "BEAR IS DRIVING!?\n";
+                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run ( command => $command, verbose => $verbose );
+                    unless ($success) {
+                        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                        &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+                    }
+                }
+            }
+        }
     }
     return(0);
@@ -826 +1093 @@
 
     my $label = "${target}.nightlyscience";
-    my $workdir = "neb://\@HOST\@.0/gpc1/${target}.nt/${workdir_date}";
+    my $workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}";
     my $obs_mode = $obsmode_list{$target};
     my $object   = $object_list{$target};
@@ -837 +1104 @@
 }
 
+sub get_dettool_parameters {
+    my $date = shift;
+    my $target = shift;
+    my $workdir_date = $date; $workdir_date =~ s%-%/%g;
+    my $trunc_date = $date; $trunc_date =~ s/-//g;
+
+    my $exp_type = $exptype_list{$target};
+    my $det_type = $dettype_list{$target};
+    my $ref_det_id = $refID_list{$target};
+    my $ref_iter = $refIter_list{$target};
+    my $det_filter = $detfilter_list{$target};
+    my $internal_filter;
+    if (defined($det_filter)) {
+        $internal_filter = $det_filter; $internal_filter =~ s/\..*//;
+        $internal_filter = '.' . $internal_filter;
+    }
+    else {
+        $internal_filter = '';
+    }
+    my $maxN = $detmax_list{$target};    
+    
+    my $lc_type = lc($exp_type);
+    my $label = "${lc_type}${internal_filter}.$trunc_date";
+    my $workdir = 'neb://@HOST@.0/' . $dbname . "/detverify.nt/${workdir_date}/${lc_type}${internal_filter}";
+    return($label,$workdir,$det_filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN);
+}
 
 sub init_gpc_db {