IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jun 22, 2010, 6:20:00 PM (16 years ago)
Author:
watersc1
Message:

Extensive update to nightly processing code:

  • Add task and automation handler to queue dqstats runs.
  • Fix ns.detrends.load trange bug.
  • Alter how cleanup automation happens to implement distribution cleanup and allow for finer grained controls in target/mode pairs.
  • Re-enable the definition of diff cleans in the config file, but set the retention time negative to disallow it.
  • Move SweetSpot from NOCLEAN to standard clean with a warp retention time of two months.

Implement new dqstats bundle code for automation, and change the acceptance criteria to be more realistic.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/automate_stacks.pl

    r28388 r28427  
    1717my $missing_tools = 0;
    1818my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
     19my $dqstatstool = can_run('dqstatstool') or (warn "Can't find dqstatstool" and $missing_tools = 1);
    1920my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
    2021my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
    … …  
    2829    exit($PS_EXIT_CONFIG_ERROR);
    2930}
    30 
    31 # my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000');
    32 # my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10',
    33 #                  'STS','SVS','SweetSpot','ThreePi');
    34 # my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02',
    35 #                  'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06',
    36 #                  'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10',
    37 #                  'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0');
    38 # my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%',
    39 #                   'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%',
    40 #                   'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%',
    41 #                   'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%');
    42 # my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1,
    43 #                   'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1,
    44 #                   'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1,
    45 #                   'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0);
    46 # my $retention_time = 9000;  # days.
    47 
    4831
    4932my $db;
    … …  
    6043my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
    6144my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
    62 my ( $check_detrends, $queue_detrends);
     45my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats);
    6346
    6447GetOptions(
    … …  
    8366    'check_stacks'         => \$check_stacks,
    8467    'queue_stacks'         => \$queue_stacks,
    85     'check_detrends'          => \$check_detrends,
    86     'queue_detrends'          => \$queue_detrends,
     68    'check_detrends'       => \$check_detrends,
     69    'queue_detrends'       => \$queue_detrends,
     70    'check_dqstats'        => \$check_dqstats,
     71    'queue_dqstats'        => \$queue_dqstats,
    8772    'check_diffs'          => \$check_diffs,
    8873    'queue_diffs'          => \$queue_diffs,
    … …  
    11196           --check_detrends       Confirm that detrend verify runs can be built.
    11297           --queue_detrends       Issue dettool commands to queue detrend verify runs.
     98           --check_dqstats        Confirm that dqstats tables can be built.
     99           --queue_dqstats        Issue dqstatstool commands to queue dqstat tables.
    113100           --check_diffs          Confirm that diffs can be done.
    114101           --queue_diffs          Issue difftool commands to queue diffs.
    … …  
    121108          ) unless
    122109    defined $check_registration or defined $define_burntool or defined $queue_burntool or
    123     defined $queue_chips or defined $queue_stacks or $queue_detrends or
    124     defined $check_chips or defined $check_stacks or $check_detrends or
     110    defined $queue_chips or defined $queue_stacks or $queue_detrends or $queue_dqstats or
     111    defined $check_chips or defined $check_stacks or $check_detrends or $check_dqstats or
    125112    defined $test_mode or defined $clean_old or defined $check_mode;
    126113
    … …  
    132119my %object_list = ();
    133120my %comment_list= ();
     121my %cleanmods_list = ();
    134122my %stackable_list = ();
    135123my %reduction_class = ();
    … …  
    144132my %clean_retention = ();
    145133my %noclean_list = ();
     134my %clean_alternate = ();
    146135# Grab the configuration data.
    147136my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
    … …  
    150139unless ($success) {
    151140    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    152     &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR);
     141    &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR);
    153142}
    154143
    … …  
    169158                $clean_retention{$this_mode} = ${ $mentry }{value};
    170159            }
     160            elsif (${ $mentry }{name} eq 'ALTERNATE_CMD') {
     161                $clean_alternate{$this_mode} = ${ $mentry }{value};
     162            }
    171163        }
    172164    }
    … …  
    202194            elsif (${ $tentry }{name} eq 'NOCLEAN') {
    203195                $noclean_list{$this_target} = ${ $tentry }{value};
     196            }
     197            else {
     198                if (exists($clean_commands{ ${ $tentry }{name} })) {
     199                    $cleanmods_list{$this_target}{${ $tentry }{name} } = ${ $tentry }{value};
     200                }
    204201            }
    205202        }
    … …  
    292289    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
    293290}
    294 if (defined($check_detrends) || defined($test_mode)) {
     291if (defined($check_dqstats) || defined($test_mode)) {
     292    $metadata_out{nsState} = 'CHECKDQSTATS';
     293    &execute_dqstats($date,"pretend");
     294    return_metadata($date);
     295    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
     296}
     297if (defined($queue_dqstats) || defined($test_mode)) {
     298    $metadata_out{nsState} = 'QUEUEDQSTATS';
     299    &execute_dqstats($date);
     300    return_metadata($date);
     301    unless (defined($test_mode)) { exit(0); }
     302}
     303if (defined($check_detrends) || defined($test_mode) || defined($check_mode)) {
    295304    $metadata_out{nsState} = 'CHECKDETRENDS';
    296305    &execute_detrends($date,"pretend");
    … …  
    300309if (defined($queue_detrends)) {
    301310    $metadata_out{nsState} = 'QUEUEDETRENDS';
    302     &execute_detrends($date,"pretend");
     311    &execute_detrends($date);
    303312    return_metadata($date);
    304313    exit(0);
    … …  
    613622}
    614623
    615 
    616 
    617624sub chip_queue {
    618625    my $date = shift;
    … …  
    660667}
    661668
     669#
     670# DQstats
     671################################################################################
     672
     673sub construct_dqstats_cmd {
     674    my $date = shift;
     675
     676    my $select = "-dateobs_end ${date}T23:59:59 ";
     677
     678    my $cmd = "$dqstatstool";
     679    $cmd .= " -simple -dbname $dbname -definebyquery ";
     680    $cmd .= " $select ";
     681    $cmd .= " -label %.nightlyscience ";
     682    $cmd .= " -set_label dqstats.nightlyscience ";
     683    if ($debug == 1) {
     684        $cmd .= ' -pretend ';
     685    }
     686    print STDERR "$cmd\n";
     687    return($cmd);
     688}
     689
     690sub pre_dqstats_queue {
     691    my $date = shift;
     692   
     693#     my $db = init_gpc_db();
     694#     my $trunc_date = $date; $trunc_date =~ s/-//g;
     695
     696#     my $where = " label LIKE '%.nightlyscience' AND data_group' ";
     697#     my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new' OR state = 'cleaned') AND $where ";
     698#     my $cam_sth = "SELECT * from camRun WHERE state = 'full' AND $where ";
     699
     700#     my $chip_ref = $db->selectall_arrayref( $chip_sth );
     701#     my $cam_ref = $db->selectall_arrayref( $cam_sth );
     702   
     703    my $command = construct_dqstats_cmd($date) . ' -pretend ';
     704    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     705        run ( command => $command, verbose => $verbose );
     706    unless ($success) {
     707        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     708        &my_die("Unable to perform dqstatstool: $error_code",1,1,,$date, $PS_EXIT_SYS_ERROR);
     709    }
     710   
     711    my @input_exposures = split /\n/, (join '', @$stdout_buf);
     712
     713    return($#input_exposures + 1,1,1); # $#{ $chip_ref } + 1, $#{ $cam_ref } + 1);
     714}
     715 
     716sub dqstats_queue {
     717    my $date = shift;
     718
     719    my $command = construct_dqstatstool_cmd($date);
     720    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     721        run ( command => $command, verbose => $verbose );
     722    unless ($success) {
     723        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     724        &my_die("Unable to perform dqstatstool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     725    }
     726    return(0);
     727
     728}
     729sub execute_dqstats {
     730    my $date = shift;
     731    my $pretend = shift;
     732    my ($Nexposures,$Nchips,$Ncams) = pre_dqstats_queue($date);
     733    if ($Nexposures == 0) {
     734        print STDERR "execute_dqstats: No exposures on date $date.\n";
     735    }
     736    elsif ($Ncams != $Nchips) {
     737        print STDERR "execute_dqstats: Not done processing data through camera stage.\n";
     738    }
     739    else {
     740        unless(defined($pretend)) {
     741            dqstats_queue($date);
     742        }
     743    }
     744}   
    662745#
    663746# Detrend verification
    … …  
    735818    $metadata_out{nsState} = 'DETREND_QUEUED';
    736819    return(0);
    737 }
    738 
     820
     821}
    739822sub execute_detrends {
    740823    my $date = shift;
    … …  
    754837    if ($exposures == 0) {
    755838        $metadata_out{nsState} = 'DETREND_DROP';
     839    }
     840    if (($metadata_out{nsState} eq 'CHECKDETRENDS') && ($exposures > 0)) {
     841        $metadata_out{nsState} eq 'QUEUE_DETRENDS';
    756842    }
    757843}
    … …  
    9141000
    9151001    my $command = $clean_commands{$mode};
    916     my $retention_time = $clean_retention{$mode};
     1002    my $retention_time;
     1003    if (exists($cleanmods_list{$target}{$mode})) {
     1004        $retention_time = $cleanmods_list{$target}{$mode};
     1005    }
     1006    else {
     1007        $retention_time = $clean_retention{$mode};
     1008    }
     1009    if ($retention_time <= 0) {
     1010        return("no clean","true");
     1011    }
    9171012
    9181013    my ($year,$month,$day) = split /-/,$date;
    … …  
    9201015                               hour => 0, minute => 0, second => 0, nanosecond => 0,
    9211016                               time_zone => 'Pacific/Honolulu');
     1017       
    9221018    $dt->subtract(days => $retention_time);
    9231019    my $cleaning_date = $dt->ymd;
    … …  
    9251021    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);
    9261022    my $args = $command;
    927     $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
     1023    if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) {
     1024        $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -time_stamp_end $cleaning_date ";
     1025    }
     1026    else {
     1027        $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
     1028    }
    9281029    if ($debug == 1) {
    9291030        $args .= ' -pretend ';
    … …  
    9361037    my $pretend = shift;
    9371038
    938     foreach my $mode (keys (%clean_commands)) {
    939         foreach my $target (@target_list) {
    940             if (exists($noclean_list{$target})) {
     1039    foreach my $mode (sort (keys (%clean_commands))) {
     1040        if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) {
     1041            my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode);
     1042            if ($cleaning_date eq 'no clean') {
    9411043                next;
    9421044            }
    943             my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
    944 
    9451045            print STDERR "$command\n";
    9461046            if (!(defined($pretend) || $debug == 1)) {
    … …  
    9511051                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    9521052                    &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     1053                }
     1054            }
     1055        }
     1056        else {
     1057            foreach my $target (@target_list) {
     1058                if (exists($noclean_list{$target})) {
     1059                    next;
     1060                }
     1061                my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
     1062                if ($cleaning_date eq 'no clean') {
     1063                    next;
     1064                }               
     1065                print STDERR "$command\n";
     1066                if (!(defined($pretend) || $debug == 1)) {
     1067#           print STDERR "BEAR IS DRIVING!?\n";
     1068                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1069                        run ( command => $command, verbose => $verbose );
     1070                    unless ($success) {
     1071                        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1072                        &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     1073                    }
    9531074                }
    9541075            }
Note: See TracChangeset for help on using the changeset viewer.