IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 28427


Ignore:
Timestamp:
Jun 22, 2010, 6:20:00 PM (16 years ago)
Author:
watersc1
Message:

Extensive update to nightly processing code:

  • Add task and automation handler to queue dqstats runs.
  • Fix ns.detrends.load trange bug.
  • Alter how cleanup automation happens to implement distribution cleanup and allow for finer grained controls in target/mode pairs.
  • Re-enable the definition of diff cleans in the config file, but set the retention time negative to disallow it.
  • Move SweetSpot from NOCLEAN to standard clean with a warp retention time of two months.

Implement new dqstats bundle code for automation, and change the acceptance criteria to be more realistic.

Location:
trunk
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/automate_stacks.pl

    r28388 r28427  
    1717my $missing_tools = 0;
    1818my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
     19my $dqstatstool = can_run('dqstatstool') or (warn "Can't find dqstatstool" and $missing_tools = 1);
    1920my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
    2021my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
    … …  
    2829    exit($PS_EXIT_CONFIG_ERROR);
    2930}
    30 
    31 # my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000');
    32 # my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10',
    33 #                  'STS','SVS','SweetSpot','ThreePi');
    34 # my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02',
    35 #                  'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06',
    36 #                  'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10',
    37 #                  'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0');
    38 # my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%',
    39 #                   'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%',
    40 #                   'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%',
    41 #                   'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%');
    42 # my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1,
    43 #                   'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1,
    44 #                   'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1,
    45 #                   'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0);
    46 # my $retention_time = 9000;  # days.
    47 
    4831
    4932my $db;
    … …  
    6043my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
    6144my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
    62 my ( $check_detrends, $queue_detrends);
     45my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats);
    6346
    6447GetOptions(
    … …  
    8366    'check_stacks'         => \$check_stacks,
    8467    'queue_stacks'         => \$queue_stacks,
    85     'check_detrends'          => \$check_detrends,
    86     'queue_detrends'          => \$queue_detrends,
     68    'check_detrends'       => \$check_detrends,
     69    'queue_detrends'       => \$queue_detrends,
     70    'check_dqstats'        => \$check_dqstats,
     71    'queue_dqstats'        => \$queue_dqstats,
    8772    'check_diffs'          => \$check_diffs,
    8873    'queue_diffs'          => \$queue_diffs,
    … …  
    11196           --check_detrends       Confirm that detrend verify runs can be built.
    11297           --queue_detrends       Issue dettool commands to queue detrend verify runs.
     98           --check_dqstats        Confirm that dqstats tables can be built.
     99           --queue_dqstats        Issue dqstatstool commands to queue dqstat tables.
    113100           --check_diffs          Confirm that diffs can be done.
    114101           --queue_diffs          Issue difftool commands to queue diffs.
    … …  
    121108          ) unless
    122109    defined $check_registration or defined $define_burntool or defined $queue_burntool or
    123     defined $queue_chips or defined $queue_stacks or $queue_detrends or
    124     defined $check_chips or defined $check_stacks or $check_detrends or
     110    defined $queue_chips or defined $queue_stacks or $queue_detrends or $queue_dqstats or
     111    defined $check_chips or defined $check_stacks or $check_detrends or $check_dqstats or
    125112    defined $test_mode or defined $clean_old or defined $check_mode;
    126113
    … …  
    132119my %object_list = ();
    133120my %comment_list= ();
     121my %cleanmods_list = ();
    134122my %stackable_list = ();
    135123my %reduction_class = ();
    … …  
    144132my %clean_retention = ();
    145133my %noclean_list = ();
     134my %clean_alternate = ();
    146135# Grab the configuration data.
    147136my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
    … …  
    150139unless ($success) {
    151140    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    152     &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR);
     141    &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR);
    153142}
    154143
    … …  
    169158                $clean_retention{$this_mode} = ${ $mentry }{value};
    170159            }
     160            elsif (${ $mentry }{name} eq 'ALTERNATE_CMD') {
     161                $clean_alternate{$this_mode} = ${ $mentry }{value};
     162            }
    171163        }
    172164    }
    … …  
    202194            elsif (${ $tentry }{name} eq 'NOCLEAN') {
    203195                $noclean_list{$this_target} = ${ $tentry }{value};
     196            }
     197            else {
     198                if (exists($clean_commands{ ${ $tentry }{name} })) {
     199                    $cleanmods_list{$this_target}{${ $tentry }{name} } = ${ $tentry }{value};
     200                }
    204201            }
    205202        }
    … …  
    292289    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
    293290}
    294 if (defined($check_detrends) || defined($test_mode)) {
     291if (defined($check_dqstats) || defined($test_mode)) {
     292    $metadata_out{nsState} = 'CHECKDQSTATS';
     293    &execute_dqstats($date,"pretend");
     294    return_metadata($date);
     295    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
     296}
     297if (defined($queue_dqstats) || defined($test_mode)) {
     298    $metadata_out{nsState} = 'QUEUEDQSTATS';
     299    &execute_dqstats($date);
     300    return_metadata($date);
     301    unless (defined($test_mode)) { exit(0); }
     302}
     303if (defined($check_detrends) || defined($test_mode) || defined($check_mode)) {
    295304    $metadata_out{nsState} = 'CHECKDETRENDS';
    296305    &execute_detrends($date,"pretend");
    … …  
    300309if (defined($queue_detrends)) {
    301310    $metadata_out{nsState} = 'QUEUEDETRENDS';
    302     &execute_detrends($date,"pretend");
     311    &execute_detrends($date);
    303312    return_metadata($date);
    304313    exit(0);
    … …  
    613622}
    614623
    615 
    616 
    617624sub chip_queue {
    618625    my $date = shift;
    … …  
    660667}
    661668
     669#
     670# DQstats
     671################################################################################
     672
     673sub construct_dqstats_cmd {
     674    my $date = shift;
     675
     676    my $select = "-dateobs_end ${date}T23:59:59 ";
     677
     678    my $cmd = "$dqstatstool";
     679    $cmd .= " -simple -dbname $dbname -definebyquery ";
     680    $cmd .= " $select ";
     681    $cmd .= " -label %.nightlyscience ";
     682    $cmd .= " -set_label dqstats.nightlyscience ";
     683    if ($debug == 1) {
     684        $cmd .= ' -pretend ';
     685    }
     686    print STDERR "$cmd\n";
     687    return($cmd);
     688}
     689
     690sub pre_dqstats_queue {
     691    my $date = shift;
     692   
     693#     my $db = init_gpc_db();
     694#     my $trunc_date = $date; $trunc_date =~ s/-//g;
     695
     696#     my $where = " label LIKE '%.nightlyscience' AND data_group' ";
     697#     my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new' OR state = 'cleaned') AND $where ";
     698#     my $cam_sth = "SELECT * from camRun WHERE state = 'full' AND $where ";
     699
     700#     my $chip_ref = $db->selectall_arrayref( $chip_sth );
     701#     my $cam_ref = $db->selectall_arrayref( $cam_sth );
     702   
     703    my $command = construct_dqstats_cmd($date) . ' -pretend ';
     704    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     705        run ( command => $command, verbose => $verbose );
     706    unless ($success) {
     707        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     708        &my_die("Unable to perform dqstatstool: $error_code",1,1,,$date, $PS_EXIT_SYS_ERROR);
     709    }
     710   
     711    my @input_exposures = split /\n/, (join '', @$stdout_buf);
     712
     713    return($#input_exposures + 1,1,1); # $#{ $chip_ref } + 1, $#{ $cam_ref } + 1);
     714}
     715 
     716sub dqstats_queue {
     717    my $date = shift;
     718
     719    my $command = construct_dqstatstool_cmd($date);
     720    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     721        run ( command => $command, verbose => $verbose );
     722    unless ($success) {
     723        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     724        &my_die("Unable to perform dqstatstool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     725    }
     726    return(0);
     727
     728}
     729sub execute_dqstats {
     730    my $date = shift;
     731    my $pretend = shift;
     732    my ($Nexposures,$Nchips,$Ncams) = pre_dqstats_queue($date);
     733    if ($Nexposures == 0) {
     734        print STDERR "execute_dqstats: No exposures on date $date.\n";
     735    }
     736    elsif ($Ncams != $Nchips) {
     737        print STDERR "execute_dqstats: Not done processing data through camera stage.\n";
     738    }
     739    else {
     740        unless(defined($pretend)) {
     741            dqstats_queue($date);
     742        }
     743    }
     744}   
    662745#
    663746# Detrend verification
    … …  
    735818    $metadata_out{nsState} = 'DETREND_QUEUED';
    736819    return(0);
    737 }
    738 
     820
     821}
    739822sub execute_detrends {
    740823    my $date = shift;
    … …  
    754837    if ($exposures == 0) {
    755838        $metadata_out{nsState} = 'DETREND_DROP';
     839    }
     840    if (($metadata_out{nsState} eq 'CHECKDETRENDS') && ($exposures > 0)) {
     841        $metadata_out{nsState} eq 'QUEUE_DETRENDS';
    756842    }
    757843}
    … …  
    9141000
    9151001    my $command = $clean_commands{$mode};
    916     my $retention_time = $clean_retention{$mode};
     1002    my $retention_time;
     1003    if (exists($cleanmods_list{$target}{$mode})) {
     1004        $retention_time = $cleanmods_list{$target}{$mode};
     1005    }
     1006    else {
     1007        $retention_time = $clean_retention{$mode};
     1008    }
     1009    if ($retention_time <= 0) {
     1010        return("no clean","true");
     1011    }
    9171012
    9181013    my ($year,$month,$day) = split /-/,$date;
    … …  
    9201015                               hour => 0, minute => 0, second => 0, nanosecond => 0,
    9211016                               time_zone => 'Pacific/Honolulu');
     1017       
    9221018    $dt->subtract(days => $retention_time);
    9231019    my $cleaning_date = $dt->ymd;
    … …  
    9251021    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);
    9261022    my $args = $command;
    927     $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
     1023    if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) {
     1024        $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -time_stamp_end $cleaning_date ";
     1025    }
     1026    else {
     1027        $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
     1028    }
    9281029    if ($debug == 1) {
    9291030        $args .= ' -pretend ';
    … …  
    9361037    my $pretend = shift;
    9371038
    938     foreach my $mode (keys (%clean_commands)) {
    939         foreach my $target (@target_list) {
    940             if (exists($noclean_list{$target})) {
     1039    foreach my $mode (sort (keys (%clean_commands))) {
     1040        if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) {
     1041            my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode);
     1042            if ($cleaning_date eq 'no clean') {
    9411043                next;
    9421044            }
    943             my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
    944 
    9451045            print STDERR "$command\n";
    9461046            if (!(defined($pretend) || $debug == 1)) {
    … …  
    9511051                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    9521052                    &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     1053                }
     1054            }
     1055        }
     1056        else {
     1057            foreach my $target (@target_list) {
     1058                if (exists($noclean_list{$target})) {
     1059                    next;
     1060                }
     1061                my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
     1062                if ($cleaning_date eq 'no clean') {
     1063                    next;
     1064                }               
     1065                print STDERR "$command\n";
     1066                if (!(defined($pretend) || $debug == 1)) {
     1067#           print STDERR "BEAR IS DRIVING!?\n";
     1068                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1069                        run ( command => $command, verbose => $verbose );
     1070                    unless ($success) {
     1071                        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1072                        &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     1073                    }
    9531074                }
    9541075            }

  • trunk/ippTasks/nightly_stacks.pro

    r28416 r28427  
    66    ns.initday.on
    77    ns.detrends.off
     8    ns.dqstats.on
    89    ns.registration.on
    910    ns.burntool.on
    … …  
    5152end
    5253
     54macro ns.dqstats.on
     55  task ns.dqstats.load
     56    active true
     57  end
     58end
     59
    5360macro ns.registration.on
    5461  task ns.registration.load
    … …  
    9299macro ns.detrends.off
    93100  task ns.detrends.load
     101    active false
     102  end
     103end
     104
     105macro ns.dqstats.off
     106  task ns.dqstats.load
    94107    active false
    95108  end
    … …  
    227240  task.exit       0
    228241    # nothign to do here
     242  end
     243  # locked list
     244  task.exit       default
     245    showcommand failure
     246  end
     247  task.exit       crash
     248    showcommand crash
     249  end
     250  # operation times out
     251  task.exit       timeout
     252    showcommand timeout
     253  end
     254end
     255
     256#
     257# Queue dqstats runs
     258#
     259task              ns.dqstats.load
     260  host            local
     261  periods         -poll 3600
     262  periods         -exec $LOADEXEC
     263  periods         -timeout 30
     264  trange          22:00:00 23:59:59
     265  trange          00:00:00 10:00:00
     266  npending        1
     267
     268  task.exec
     269    stdout $LOGDIR/ns.dqstats.log
     270    stderr $LOGDIR/ns.dqstats.log
     271    $today = `date +%Y-%m-%d`
     272
     273    command automate_stacks.pl --queue_dqstats --date $today
     274  end
     275
     276  task.exit       0
     277    # nothing to do here
    229278  end
    230279  # locked list

  • trunk/ippTools/src/dqstatstool.c

    r27304 r28427  
    8585
    8686  if (label) {
    87     PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "==");
     87    pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "LIKE"); // define using cam label
    8888  }
    8989  // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE
    … …  
    654654        psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname);
    655655        return(false);
    656       }     
     656      }
     657      psString filter = psMetadataLookupStr(&status,rule,"FILTER");
     658      if (!status) {
     659        status = true;
     660      }
     661      if (filter) {
     662        psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER");
     663        if (strcmp(filter,imfilter) != 0) {
     664          continue;
     665        }
     666      }
    657667      // Not happy with this being set to a F32. Can this ever be something else?
    658668      psF32 value      = psMetadataLookupF32(&status,tableRow,colname);

  • trunk/ippconfig/recipes/dqStatsTool.config

    r27304 r28427  
    77        COLNAME STR FWHM_MAJOR
    88        MINIMUM F32 0.0
    9         MAXIMUM F32 5.0
     9        MAXIMUM F32 9.3            # 90-percentile 2010-02-14 through 2010-06-20 = 2.4"
    1010    END
    1111    RULE METADATA
    1212        COLNAME STR FWHM_MINOR
    1313        MINIMUM F32 0.0
    14         MAXIMUM F32 5.0
     14        MAXIMUM F32 8.44           # 90-percentile 2010-02-14 through 2010-06-20 = 2.18"
     15    END
     16    RULE METADATA
     17         COLNAME STR ZEROPOINT
     18         FILTER  STR g.00000
     19         MINIMUM F32 24.3051           # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes
     20         MAXIMUM F32 99.9999           # We refuse to believe in negative clouds.
     21    END
     22    RULE METADATA
     23         COLNAME STR ZEROPOINT
     24         FILTER  STR r.00000
     25         MINIMUM F32 24.5294           # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes
     26         MAXIMUM F32 99.9999           # We refuse to believe in negative clouds.
     27    END
     28    RULE METADATA
     29         COLNAME STR ZEROPOINT
     30         FILTER  STR i.00000
     31         MINIMUM F32 24.4641           # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes
     32         MAXIMUM F32 99.9999           # We refuse to believe in negative clouds.
     33    END
     34    RULE METADATA
     35         COLNAME STR ZEROPOINT
     36         FILTER  STR z.00000
     37         MINIMUM F32 24.1128           # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes
     38         MAXIMUM F32 99.9999           # We refuse to believe in negative clouds.
     39    END
     40    RULE METADATA
     41         COLNAME STR ZEROPOINT
     42         FILTER  STR y.00000
     43         MINIMUM F32 23.1503           # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes
     44         MAXIMUM F32 99.9999           # We refuse to believe in negative clouds.
     45    END
     46    RULE METADATA
     47         COLNAME STR ZEROPOINT
     48         FILTER  STR w.00000
     49         MINIMUM F32 25.8944           # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes
     50         MAXIMUM F32 99.9999           # We refuse to believe in negative clouds.
    1551    END
    1652#    TYPE VAL COLNAME      MINIMUM        MAXIMUM

  • trunk/ippconfig/recipes/nightly_science.config

    r28408 r28427  
    66  COMMAND STR chiptool
    77  RETENTION_TIME U16 30
     8  ALTERNATE_CMD BOOL F
    89END
    910CLEAN_MODES METADATA
    … …  
    1112  COMMAND STR warptool
    1213  RETENTION_TIME U16 7
    13 END
    14 # CLEAN_MODES METADATA
    15 #  MODE STR DIFF
    16 #  COMMAND STR difftool
    17 #  RETENTION_TIME U16 30
    18 # END
     14  ALTERNATE_CMD BOOL F
     15END
     16CLEAN_MODES METADATA
     17  MODE STR DIFF
     18  COMMAND STR difftool
     19# RETENTION_TIME U16 30
     20  RETENTION_TIME S16 -1
     21  ALTERNATE_CMD BOOL F
     22END
     23CLEAN_MODES METADATA
     24  MODE STR DIST
     25  COMMAND STR disttool
     26  RETENTION_TIME S16 7
     27  ALTERNATE_CMD BOOL T
     28END
    1929
    2030FILTERS MULTI
    … …  
    128138# OBJECT STR SS
    129139  STACKABLE BOOL FALSE
    130   NOCLEAN BOOL TRUE
     140#  NOCLEAN BOOL TRUE
     141  WARP S16 60
    131142END
    132143TARGETS METADATA
    … …  
    293304DETRENDS METADATA
    294305  NAME     STR DARK
    295   DETTYPE     STR DARK
     306  DETTYPE  STR DARK
    296307  EXPTYPE  STR dark
    297308  REF_ID   S64 299
    … …  
    303314DETRENDS METADATA
    304315  NAME     STR FLAT_G
    305   DETTYPE     STR FLAT
     316  DETTYPE  STR FLAT
    306317  EXPTYPE  STR skyflat
    307318  REF_ID   S64 300
    … …  
    313324DETRENDS METADATA
    314325  NAME     STR FLAT_R
    315   DETTYPE     STR FLAT
     326  DETTYPE  STR FLAT
    316327  EXPTYPE  STR skyflat
    317328  REF_ID   S64 301
    … …  
    323334DETRENDS METADATA
    324335  NAME     STR FLAT_I
    325   DETTYPE     STR FLAT
     336  DETTYPE  STR FLAT
    326337  EXPTYPE  STR skyflat
    327338  REF_ID   S64 302
    … …  
    333344DETRENDS METADATA
    334345  NAME     STR FLAT_Z
    335   DETTYPE     STR FLAT
     346  DETTYPE  STR FLAT
    336347  EXPTYPE  STR skyflat
    337348  REF_ID   S64 303
    … …  
    343354DETRENDS METADATA
    344355  NAME     STR FLAT_Y
    345   DETTYPE     STR FLAT
     356  DETTYPE  STR FLAT
    346357  EXPTYPE  STR skyflat
    347358  REF_ID   S64 304
    … …  
    353364DETRENDS METADATA
    354365  NAME     STR FLAT_W
    355   DETTYPE     STR FLAT
     366  DETTYPE  STR FLAT
    356367  EXPTYPE  STR skyflat
    357368  REF_ID   S64 305
Note: See TracChangeset for help on using the changeset viewer.