Index: /branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl (revision 30014) +++ /branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl (revision 30014) @@ -0,0 +1,1548 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use DateTime; + +# +# Set up +################################################################################ + +my $missing_tools = 0; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $dqstatstool = can_run('dqstatstool') or (warn "Can't find dqstatstool" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1); +my $moondata = can_run('moondata') or (warn "Can't find moondata" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $db; +my $debug = 0; +my %metadata_out = (); +$metadata_out{nsState} = 'UNDEFINEDERROR'; +my $loghead; +chomp($loghead = `date`); +print STDERR 'Starting: ' . $loghead . ' ' . $0 . ' ' . (join ' ', @ARGV) . "\n"; + +# Grab options +my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual); +my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode); +my ( $registration_status, $burntool_status, $observing_status); +my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); +my ( $check_stacks, $queue_stacks, $check_sweetspot, $queue_sweetspot, $check_diffs, $queue_diffs, $clean_old); +my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats); +my ( $confirm_stacks, $check_confirm_stacks ); + +GetOptions( + 'help|h' => \$help, + 'date=s' => \$date, # night to make stacks for + 'camera=s' => \$camera, # camera + 'dbname=s' => \$dbname, # Database name + 'logfile=s' => \$logfile, + 'verbose' => \$verbose, + 'debug' => \$debug, + 'test_mode' => \$test_mode, + 'force_stack_count' => \$force_stack_count, + 'this_target_only=s' => \$this_target_only, + 'this_filter_only=s' => \$this_filter_only, + 'this_mode_only=s' => \$this_mode_only, + 'registraion_status' => \$registration_status, + 'burntool_status' => \$burntool_status, + 'check_stacks' => \$check_stacks, + 'queue_stacks' => \$queue_stacks, + 'confirm_stacks' => \$confirm_stacks, + 'check_confirm_stacks' => \$check_confirm_stacks, + 'check_diffs' => \$check_diffs, + 'queue_diffs' => \$queue_diffs, + 'check_sweetspot' => \$check_sweetspot, + 'queue_sweetspot' => \$queue_sweetspot, + 'check_detrends' => \$check_detrends, + 'queue_detrends' => \$queue_detrends, + 'check_dqstats' => \$check_dqstats, + 'queue_dqstats' => \$queue_dqstats, + 'clean_old' => \$clean_old, + ) or pod2usage ( 2 ); +pod2usage( -msg => +"USAGE: automate_stacks.pl + Options: + --help This help. + --date YYYY-MM-DD Work on this date (defaults to today GMT). + --camera Default GPC1. + --dbname Default gpc1. + --verbose + --force_stack_count Force the chip/warp counts. + --this_target_only Process only a single target. + --this_filter_only Process only a single filter. + --this_mode_only Process only a single clean mode. + --burntool_status Display Nexp Nimfile Nburntooled Nqueued for check_chips. + Modes: + --check_stacks Confirm that stacks can be built. + --queue_stacks Issue stacktool commands to queue stacks. + --check_diffs Confirm that diffs can be done. + --queue_diffs Issue difftool commands to queue diffs. + --check_sweetspot See if we should queue SweetSpot stacks. + --queue_sweetspot Issue stacktool commands to queue SweetSpot stacks. + --check_detrends Confirm that detrend verify runs can be built. + --queue_detrends Issue dettool commands to queue detrend verify runs. + --check_dqstats Confirm that dqstats tables can be built. + --queue_dqstats Issue dqstatstool commands to queue dqstat tables. + --clean_old Mark old data 'goto_cleanup'.\n", + -exitval => 2, ) if (defined($help)); +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks", + -exitval => 3, + ) unless + defined $check_registration or defined $define_burntool or defined $queue_burntool or + defined $queue_chips or defined $queue_stacks or $queue_sweetspot or $queue_detrends or $queue_dqstats or + defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or + defined $test_mode or defined $clean_old or defined $check_mode or + defined $confirm_stacks or defined $burntool_status; + +# Configurable parameters from our config file. +my %science_config = (); +my %clean_config = (); +my %detrend_config = (); +my %eon_config = (); +my %cleanmods_list = (); +my %macro_config = (); +my @unrecoverable_quality = (); +my @filter_list = (); + +# Grab the configuration data. +my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; +my $metadata = $mdcParser->parse(join "", @$stdout_buf); +foreach my $entry (@{ $metadata }) { + if (@{ $entry }{name} eq 'CLEAN_MODES') { + my @mode_data = @{ ${ $entry }{value} }; + my $this_mode = ''; + foreach my $mentry (@mode_data) { + if (${ $mentry }{name} eq 'MODE') { + $this_mode = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'COMMAND') { + $clean_config{$this_mode}{COMMAND} = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'RETENTION_TIME') { + $clean_config{$this_mode}{RETENTION_TIME} = ${ $mentry }{value}; + } + } + } + elsif (${ $entry }{name} eq 'FILTERS') { + push @filter_list, ${ $entry }{value}; + } + elsif (${ $entry }{name} eq 'UNRECOVERABLE_QUALITY') { + push @unrecoverable_quality, ${ $entry }{value}; + } + elsif (${ $entry }{name} eq 'MACRO_DEFINITIONS') { + my @macro_data = @{ ${ $entry }{value} }; + my $this_mode = ''; + foreach my $mentry (@macro_data) { + if (${ $mentry }{name} eq 'PROC_MODE') { + $this_mode = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'MACRO') { + $macro_config{$this_mode} = ${ $mentry }{value}; + } + } + } + elsif (${ $entry }{name} eq 'TARGETS') { + my @target_data = @{ ${ $entry }{value} }; + my $this_target = ''; + foreach my $tentry (@target_data) { + if (${ $tentry }{name} eq 'NAME') { + $this_target = ${ $tentry }{value}; + $science_config{$this_target}{DISTRIBUTION} = $this_target; # Set the default dist_group + } + else { + $science_config{$this_target}{ ${ $tentry }{name} } = ${ $tentry }{value}; + } + } + } + elsif (${ $entry }{name} eq 'DETRENDS') { + my @detrend_data = @{ ${ $entry }{value} }; + my $this_detrend = ''; + foreach my $dentry (@detrend_data) { + if (${ $dentry }{name} eq 'NAME') { + $this_detrend = ${ $dentry }{value}; + } + else { + $detrend_config{$this_detrend}{ ${ $dentry }{name} } = ${ $dentry }{value}; + } + } + } + elsif (${ $entry }{name} eq 'END_OF_NIGHT') { + my @eon_data = @{ ${ $entry }{value} }; + my $this_eon = ''; + foreach my $tentry (@eon_data) { + if (${ $tentry }{name} eq 'NAME') { + $this_eon = ${ $tentry }{value}; + } + else { + $eon_config{$this_eon}{ ${ $tentry }{name} } = ${ $tentry }{value}; + } + } + } +} + +if (defined($date)) { + my $time; + if ($date =~ / /) { + ($date,$time) = split / /, $date; + } + elsif ($date =~ /T/) { + ($date,$time) = split /T/, $date; + } + my ($year,$month,$day) = split /-/, $date; + if (defined($time)) { + my ($hour,$min,$sec) = split /\:/, $time; #/; + $datetime = DateTime->new( year => $year, + month => $month, + day => $day, + hour => $hour, + min => $min, + second => $sec, + time_zone => 'UTC'); + } + else { + $datetime = DateTime->now(time_zone => 'UTC'); + $datetime->set_year($year); + $datetime->set_month($year); + $datetime->set_day($year); + } +} +else { + $datetime = DateTime->now(time_zone => 'UTC'); # time_zone => 'Pacific/Honolulu'); + $date = $datetime->ymd(); +} +unless(defined($camera)) { + $camera = 'GPC1'; +} +unless(defined($dbname)) { + $dbname = 'gpc1'; +} + +if (defined($this_target_only)) { + foreach my $t (keys %science_config) { + if ($t ne $this_target_only) { + undef($science_config{$t}); + } + } + die("$this_target_only is invalid.") if (scalar keys %science_config < 1); +} + +if (defined($this_filter_only)) { + foreach my $t (@filter_list) { + if ($t eq $this_filter_only) { + @filter_list = ($this_filter_only); + last; + } + } + die("$this_filter_only is invalid.") if ($#filter_list != 0); +} + +if (defined($this_mode_only)) { + foreach my $t (keys %clean_config) { + if ($t ne $this_mode_only) { + undef($clean_config{$t}); + } + } + die("$this_mode_only is invalid.") if (scalar keys %clean_config < 1); +} + +# +# Mode selection +################################################################################ +if (defined($test_mode)) { + $debug = 1; + my $z; + my $v; + foreach $z (sort (keys %science_config)) { + foreach $v (keys %{ $science_config{$z} }) { + print "SCIENCE: $z $v $science_config{$z}{$v}\n"; + } + } + foreach $z (sort (keys %detrend_config)) { + foreach $v (keys %{ $detrend_config{$z} }) { + print "DETREND: $z $v $detrend_config{$z}{$v}\n"; + } + } + foreach $z (keys (%clean_config)) { + foreach $v (keys %{ $clean_config{$z} }) { + print "CLEAN: $z $v $clean_config{$z}{$v}\n"; + } + } + foreach $z (@filter_list) { + print "FILTER: $z\n"; + } + foreach $z (keys (%macro_config)) { + print "MACROS: $z $macro_config{$z}\n"; + } +} +if (defined($registration_status) || defined($test_mode)) { + &check_summit_copy($date); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} + +if (defined($burntool_status) || defined($test_mode)) { + &burntool_status($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($observing_status) || defined($test_mode)) { + $metadata_out{nsState} = &get_observing_state($date); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} + +if (defined($check_dqstats) || defined($test_mode)) { + &execute_dqstats($date,"pretend"); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($queue_dqstats)) { + &execute_dqstats($date); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +exit(10); +if (defined($check_detrends) || defined($test_mode)) { + $metadata_out{nsState} = 'CHECKDETRENDS'; + &execute_detrends($date,"pretend"); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($queue_detrends)) { + $metadata_out{nsState} = 'QUEUEDETRENDS'; + &execute_detrends($date); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} + +if (defined($check_stacks) || defined($test_mode)) { + $metadata_out{nsState} = 'TOWARP'; + &execute_stacks($date,"pretend"); + if ($metadata_out{nsState} eq 'FORCETOWARP') { + $metadata_out{nsState} = 'TOWARP'; + } + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($queue_stacks)) { + $metadata_out{nsState} = 'STACKING'; + &execute_stacks($date); + if ($metadata_out{nsState} eq 'QUEUESTACKING') { + $metadata_out{nsState} = 'STACKING_POSSIBLE_ERROR'; + } + return_metadata($date); + exit(0); +} + +if (defined($check_diffs) || defined($test_mode)) { + $metadata_out{nsState} = 'TOWARP'; + &execute_diffs($date,"pretend"); + if ($metadata_out{nsState} eq 'FORCETOWARP') { + $metadata_out{nsState} = 'TOWARP'; + } + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($queue_diffs)) { + $metadata_out{nsState} = 'STACKING'; + &execute_diffs($date); + if ($metadata_out{nsState} eq 'QUEUESTACKING') { + $metadata_out{nsState} = 'STACKING_POSSIBLE_ERROR'; + } + return_metadata($date); + exit(0); +} + +if (defined($check_confirm_stacks) || defined($test_mode)) { + $metadata_out{nsState} = 'CONFIRM_STACKING'; + &confirm_stacks($date,"pretend"); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($confirm_stacks)) { + $metadata_out{nsState} = 'CONFIRM_STACKING'; + &confirm_stacks($date); + return_metadata($date); + exit(0); +} + +if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) { + $metadata_out{nsState} = 'CHECKSWEETSPOT'; + &execute_sweetspot($date,"pretend"); + return_metadata($date); + unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($queue_sweetspot)) { + $metadata_out{nsState} = 'CHECKSWEETSPOT'; + &execute_sweetspot($date); + return_metadata($date); + exit(0); +} + +if (defined($clean_old) || defined($test_mode)) { + if (defined($test_mode)) { + &execute_cleans($date,"pretend"); + } + else { + &execute_cleans($date); + } + unless (defined($test_mode)) { exit(0); } +} +exit(0); +# +# Registration +################################################################################ +# This isn't used, but might be useful to keep. + +sub check_summit_copy { + my $date = shift; + my $db = init_gpc_db(); + + # largely stolen from Bill's checkexp program. + + my $sth = " SELECT exp_name, summitExp.dateobs AS registered, rawExp.dateobs, summitExp.imfiles, "; + $sth .= " summitExp.fault AS summit_fault, pzDownloadExp.state AS download_state, "; + $sth .= " count(pzDownloadImfile.class_id) AS download_count, newExp.state AS newExp_state, newExp.exp_id, summitExp.exp_type"; + $sth .= " FROM summitExp LEFT JOIN pzDownloadExp USING(exp_name) LEFT JOIN pzDownloadImfile USING(exp_name) "; + $sth .= " LEFT JOIN newExp ON exp_name = tmp_exp_name LEFT JOIN rawExp USING(exp_id, exp_name) "; + $sth .= " WHERE date(summitExp.dateobs) >= '${date}T00:00:00' AND date(summitExp.dateobs) <= '${date}T23:59:59' "; + $sth .= " GROUP BY exp_name ORDER BY summitExp.dateobs "; + + my $data_ref = $db->selectall_arrayref( $sth ); + + my $Nsummit_exps = 0; + my $Nsummit_faults = 0; + my $Ndownload_faults = 0; + my $Nregister_faults = 0; + my $Ndownload_imfiles = 0; + my $Nsummit_imfiles = 0; +# my_trace($sth,$data_ref,$#{ $data_ref }); + + foreach my $row_ref (@{ $data_ref }) { + my ($exp_name,$registered,$dateobs,$imfiles,$summit_fault, + $download_state,$download_count,$new_state,$exp_id,$exp_type) = @{ $row_ref }; + $Nsummit_imfiles += $imfiles; + $Ndownload_imfiles += $download_count; + $Nsummit_exps++; + if ($summit_fault) { + print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault"; + if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) { + print STDERR " (but I don't care).\n"; + } + else { + print STDERR "\n"; + $Nsummit_faults++; + } + } + elsif (!$download_state or $download_state eq 'run') { + print STDERR "check_summit_copy: $date $exp_name has download_state $download_state"; + if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) { + print STDERR " (but I don't care).\n"; + } + else { + print STDERR "\n"; + $Ndownload_faults++; + } + } + elsif (!$new_state or $new_state eq 'run' ) { + print STDERR "check_summit_copy: $date $exp_name has new_state $new_state"; + if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) { + print STDERR " (but I don't care).\n"; + } + else { + print STDERR "\n"; + $Nregister_faults++; + } + } + } + + my $Nfaults = $Nsummit_faults + $Ndownload_faults + $Nregister_faults; + if ($Nsummit_exps == 0) { + print STDERR "No exposures were found on the summit for $date.\n"; + $metadata_out{nsState} = 'DROP'; + } + elsif ($Nfaults != 0) { + print STDERR "There were faults while downloading the exposures for $date.\n"; + $metadata_out{nsState} = 'NEW'; + } + elsif ($Ndownload_imfiles != $Nsummit_imfiles) { + print STDERR "Not done downloading from the summit for $date (Summit: $Nsummit_imfiles, Downloaded: $Ndownload_imfiles)\n"; + $metadata_out{nsState} = 'NEW'; + } + else { + print STDERR "Summit copy and Registration have succeeded for $date.\n"; + $metadata_out{nsState} = 'REGISTERED'; + } + + return($Nsummit_exps,$Nfaults); +} + +# +# Burntool +################################################################################ + +sub get_goodBTvalue { + my $mdcParser = PS::IPP::Metadata::Config->new; + + my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL.STATE.GOOD | uniq"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $config_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, 0, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", 0, 0, 0, $PS_EXIT_SYS_ERROR); + my $burntoolStateGood = 999; + foreach my $cfg (@$recipeData) { + if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') { + $burntoolStateGood = $cfg->{value}; + } + } + if ($burntoolStateGood == 999) { + &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, 0, 0, $PS_EXIT_SYS_ERROR); + } + return($burntoolStateGood); +} + +sub verify_burntool { + my $exp_id = shift; + my $burntoolStateGood = shift; + + my $db = init_gpc_db(); + + my $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,burntool_state,comment FROM rawImfile WHERE exp_id = $exp_id"; + my $data_ref = $db->selectall_arrayref( $sth ); + + my $Nimfiles = 0; + my $Nburntooled = 0; + + foreach my $row_ref (@{ $data_ref }) { + my ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment) = @{ $row_ref }; + $Nimfiles++; + if (abs($burntool_state) == $burntoolStateGood) { + $Nburntooled++; + } + } + return($Nimfiles,$Nburntooled); +} + +# +# Chips +################################################################################ +# This no longer queues chips, it only checks the counts and returns that. +sub construct_chiptool_cmd { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $select = "-dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; + $date =~ s/-//g; + + my $cmd = "$chiptool"; + $cmd .= " -simple -dbname $dbname -definebyquery -set_end_stage warp "; + $cmd .= " -set_label $label "; + $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; + $cmd .= " -set_tess_id $tess_id -set_data_group $data_group "; + if (defined($obs_mode)) { + $cmd .= " -obs_mode '$obs_mode' "; + } + if (defined($object)) { + $cmd .= " -object '$object' "; + } + if (defined($comment)) { + $cmd .= " -comment '$comment' "; + } + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + $cmd .= " $select "; + if ($debug == 1) { + $cmd .= " -pretend "; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub verify_uniqueness_chip { + my $exp_id = shift; + my $date = shift; + my $target = shift; + + my $db = init_gpc_db(); + $date =~ s/-//g; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $sth = "SELECT exp_id from chipRun where data_group = '$data_group' AND exp_id = $exp_id"; + my $data_ref = $db->selectall_arrayref( $sth ); + + return($#{ $data_ref } + 1); +} + +sub check_chip_status { + my $date = shift; + my $target = shift; + + my $command = construct_chiptool_cmd($date,$target) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + + my $burntoolStateGood = get_goodBTvalue(); + my $Nimfiles = 0; + my $Nburntooled = 0; + my $Nalready = 0; + my @input_exposures = split /\n/, (join '', @$stdout_buf); + foreach my $entry (@input_exposures) { + my ($exp_id, @trash) = split /\s+/, $entry; + @trash = verify_burntool($exp_id,$burntoolStateGood); + $Nimfiles += $trash[0]; + $Nburntooled += $trash[1]; + + @trash = verify_uniqueness_chip($exp_id,$date,$target); + $Nalready += $trash[0]; + } + return($#input_exposures + 1,$Nimfiles,$Nburntooled,$Nalready); +} + +sub burntool_status { + my $date = shift; + my $exposures = 0; + + foreach my $target (sort (keys %science_config)) { + my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = check_chip_status($date,$target); + print "BTSTATS: $date $target $Nexposures $Nimfiles $Nburntooled $Nalready\n"; + } +} + + +# +# DQstats +################################################################################ + +sub construct_dqstats_cmd { + my $date = shift; + + my $select = "-dateobs_end ${date}T23:59:59 "; + + my $cmd = "$dqstatstool"; + $cmd .= " -simple -dbname $dbname -definebyquery "; + $cmd .= " $select "; + $cmd .= " -label %.nightlyscience "; + $cmd .= " -set_label dqstats.nightlyscience "; + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub pre_dqstats_queue { + my $date = shift; + + my $command = construct_dqstats_cmd($date) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dqstatstool: $error_code",1,1,,$date, $PS_EXIT_SYS_ERROR); + } + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + + return($#input_exposures + 1,1,1); +} + +sub dqstats_queue { + my $date = shift; + + my $command = construct_dqstats_cmd($date); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dqstatstool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + return(0); + +} +sub execute_dqstats { + my $date = shift; + my $pretend = shift; + my ($Nexposures,$Nchips,$Ncams) = pre_dqstats_queue($date); + if ($Nexposures == 0) { + print STDERR "execute_dqstats: No exposures on date $date.\n"; + } + elsif ($Ncams != $Nchips) { + print STDERR "execute_dqstats: Not done processing data through camera stage.\n"; + } + else { + unless(defined($pretend)) { + dqstats_queue($date); + } + if (defined($pretend)) { + add_to_macro_list('check_dqstats',1,$date); + } + else { + add_to_macro_list('queue_dqstats',1,$date); + } + } +} +# # +# # Detrend verification +# ################################################################################ + +# sub construct_dettool_cmd { +# my $date = shift; +# my $target = shift; + +# my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); + +# my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; +# my $use_limits = " -use_begin ${date}T00:00:00 -use_end ${date}T23:59:59 "; +# $date =~ s/-//g; + +# my $cmd = "$dettool"; +# # $cmd .= " -pretend "; +# $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type "; +# $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter "; +# $cmd .= " $select "; +# $cmd .= " -inst $camera "; +# $cmd .= " -select_exp_type $exp_type "; +# $cmd .= " -select_filter $filter " if defined($filter); +# $cmd .= " -workdir $workdir "; +# $cmd .= " -label $label "; +# $cmd .= " $use_limits "; +# if ($maxN > 0) { +# $cmd .= " -random_subset -random_limit $maxN "; +# } +# if ($debug == 1) { +# $cmd .= ' -pretend '; +# } +# print STDERR "$cmd\n"; +# return($cmd); +# } + +# sub verify_uniqueness_detverify { +# my $date = shift; +# my $target = shift; + +# my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); + +# my $db = init_gpc_db(); +# my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter"; +# my $data_ref = $db->selectall_arrayref( $sth ); + +# return($#{ $data_ref } + 1); +# } + +# sub pre_detrend_queue { +# my $date = shift; +# my $target = shift; + +# my $command = construct_dettool_cmd($date,$target) . ' -pretend '; +# my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); +# } + +# my @input_exposures = split /\n/, (join '', @$stdout_buf); +# return($#input_exposures + 1); +# } + +# sub detrend_queue { +# my $date = shift; +# my $target = shift; + +# my $command = construct_dettool_cmd($date,$target); +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# $metadata_out{nsState} = 'DETREND_QUEUED'; +# return(0); + +# } +# sub execute_detrends { +# my $date = shift; +# my $pretend = shift; +# my $exposures = 0; +# foreach my $target (@detrend_list) { +# my ($Nexposures) = pre_detrend_queue($date,$target); +# if ($Nexposures == 0) { +# print STDERR "execute_detrends: Target $target on $date had no exposures.\n"; +# next; +# } +# $exposures++; +# unless(defined($pretend)) { +# detrend_queue($date,$target); +# } +# if (defined($pretend)) { +# add_to_macro_list('check_detrends',1,$date,$target,"dettool"); +# } +# else { +# add_to_macro_list('queue_detrends',1,$date,$target,"dettool"); +# } +# } +# if ($exposures == 0) { +# $metadata_out{nsState} = 'DETREND_DROP'; +# } +# if (($metadata_out{nsState} eq 'CHECKDETRENDS') && ($exposures > 0)) { +# $metadata_out{nsState} eq 'QUEUEDETRENDS'; +# } +# } + +# # +# # SweetSpot Stacking +# ################################################################################ +# sub construct_sweetspot_cmd { +# my $date = shift; +# my $target = 'SweetSpot'; +# my $filter = 'w.00000'; + +# my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + +# # Dateobs begin end? +# my ($dateobs_begin,$dateobs_end) = get_lunation_extent($date); +# my $select = "-select_dateobs_begin ${dateobs_begin}T00:00:00 -select_dateobs_end ${dateobs_end}T00:00:00"; +# my $cmd = "$stacktool"; +# $cmd .= " -simple -dbname $dbname -definebyquery "; +# $cmd .= " -set_label SweetSpot.refstack -select_label $label "; +# $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; +# $cmd .= " -select_filter $filter -set_data_group $data_group "; +# $cmd .= " -select_good_frac_min 0.1 -select_fwhm_major_max 8.0 "; +# $cmd .= " -min_num 7 -min_new 4"; +# $cmd .= " $select "; +# if ($debug == 1) { +# $cmd .= ' -pretend '; +# } +# print STDERR "$cmd\n"; +# return($cmd); +# } + +# sub get_lunation_extent { +# my $date = shift; +# my ($year,$month,$day) = split /-/,$date; +# my $dateobs_begin; +# my $dateobs_end; + +# my $dt = DateTime->new(year => $year, month => $month, day => $day, +# hour => 0, minute => 0, second => 0, nanosecond => 0, +# time_zone => 'Pacific/Honolulu'); +# do { +# $dt->subtract(days => 1); +# my $ymd = $dt->ymd; +# my $md_cmd = "moondata $ymd 0 0"; +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $md_cmd, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); +# } +# my @result = split /\s+/,(join "\n", @$stdout_buf); +# if (abs($result[6]) <= 0.5) { +# $dateobs_end = $ymd; +# } +# } while (!defined($dateobs_end)); + +# do { +# $dt->subtract(days => 1); +# my $ymd = $dt->ymd; +# my $md_cmd = "moondata $ymd 0 0"; +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $md_cmd, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); +# } +# my @result = split /\s+/,(join "", @$stdout_buf); +# if (abs($result[6]) <= 0.5) { +# $dateobs_begin = $ymd; +# } +# } while (!defined($dateobs_begin)); + +# return($dateobs_begin,$dateobs_end); +# } + +# sub pre_sweetspot_queue { +# my $date = shift; +# my $command = construct_sweetspot_cmd($date) . ' -pretend '; +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $command, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform sweetspot stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# my @stacks = split /\n/, (join '', @$stdout_buf); +# my $Nstacks = $#stacks + 1; + +# return(1,$Nstacks); +# } + +# sub sweetspot_queue { +# my $date = shift; +# my $command = construct_sweetspot_cmd($date); +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $command, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform sweetspot stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# my @stacks = split /\n/, (join '', @$stdout_buf); +# my $Nstacks = $#stacks + 1; + +# return(1,$Nstacks); +# } + +# sub execute_sweetspot { +# my $date = shift; +# my $pretend = shift; + +# my ($is_lunation_date,$Nstacks) = pre_sweetspot_queue($date); +# if ($Nstacks == 0) { +# print STDERR "execute_sweetspot: No new stacks to make ($Nstacks)\n"; +# $metadata_out{nsState} = 'SS_EMPTY'; +# return(); +# } +# if ($Nstacks < 10) { +# print STDERR "execute_sweetspot: Too few new stacks to make ($Nstacks)\n"; +# $metadata_out{nsState} = 'SS_FEW'; +# return(); +# } +# if ($is_lunation_date == 0) { +# print STDERR "execute_sweetspot: Invalid lunation date. ($date $is_lunation_date)\n"; +# $metadata_out{nsState} = 'SS_ERROR'; +# return(); +# } +# $metadata_out{nsState} = 'SS_QUEUE'; +# $metadata_out{nsSweetSpot} = $Nstacks; +# unless(defined($pretend)) { +# $metadata_out{nsState} = 'SS_DONE'; +# sweetspot_queue($date); +# } +# if (defined($pretend)) { +# add_to_macro_list('check_sweetspot',1,$date); +# } +# else { +# add_to_macro_list('queue_sweetspot',1,$date); +# } +# } + +# # +# # Stacking +# ################################################################################ + +# sub construct_stacktool_cmd { +# my $date = shift; +# my $target = shift; +# my $filter = shift; + +# my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + +# my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; +# $date =~ s/-//g; + +# my $cmd = "$stacktool"; +# # $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 '; # Probably silly, but I want to be safe and not overwrite +# $cmd .= " -simple -dbname $dbname -definebyquery "; +# $cmd .= " -set_label $label -select_label $label "; +# $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; +# $cmd .= " -select_filter $filter -set_data_group $data_group "; +# $cmd .= " -min_num 4"; +# $cmd .= " -select_good_frac_min 0.05"; +# $cmd .= " $select "; +# if ($debug == 1) { +# $cmd .= ' -pretend '; +# } +# print STDERR "$cmd\n"; +# return($cmd); +# } + +# sub verify_uniqueness_stack { +# my $skycell = shift; +# my $date = shift; +# my $target = shift; +# my $filter = shift; + +# my $db = init_gpc_db(); +# $date =~ s/-//g; +# my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + +# my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter' AND tess_id = '$tess_id'"; +# my $data_ref = $db->selectall_arrayref( $sth ); + +# return($#{ $data_ref } + 1); +# } + +# sub pre_stack_queue { +# my $date = shift; +# my $target = shift; +# my $filter = shift; + +# my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); +# # check warp stage == chip stage +# my $db = init_gpc_db(); + +# my $trunc_date = $date; $trunc_date =~ s/-//g; + +# my $where = " label = '$label' AND data_group = '$data_group' "; + +# my $where_possibly_faulted = $where . " AND ( " ; +# foreach my $acceptable_quality (@unrecoverable_quality) { +# $where_possibly_faulted .= " quality = $acceptable_quality OR "; +# } +# $where_possibly_faulted .= " 0 )"; + +# my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where "; +# my $cam_sth = "SELECT * from camRun JOIN camProcessedExp USING(cam_id) WHERE state = 'full' AND $where_possibly_faulted "; +# my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where "; + +# my $chip_ref = $db->selectall_arrayref( $chip_sth ); +# my $cam_ref = $db->selectall_arrayref( $cam_sth ); +# my $warp_ref = $db->selectall_arrayref( $warp_sth ); + +# # check that we will be able to queue them up +# my $command = construct_stacktool_cmd($date,$target,$filter) . ' -pretend '; +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $command, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform stacktool: $error_code", $#{ $chip_ref },$#{ $warp_ref },$date, $PS_EXIT_SYS_ERROR); +# } + +# my $Nalready = 0; + +# my @input_exposures = split /\n/, (join '', @$stdout_buf); +# foreach my $entry (@input_exposures) { +# my ($warp_tess_id,$skycell, @trash) = split /\s+/, $entry; +# @trash = verify_uniqueness_stack($skycell,$date,$target,$filter); +# $Nalready += $trash[0]; +# } + +# return($#input_exposures + 1, $#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2, $Nalready); +# } + +# sub stack_queue { +# my $date = shift; +# my $target = shift; +# my $filter = shift; + +# my $command = construct_stacktool_cmd($date,$target,$filter); +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $command, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } + + +# return(0); +# } + +# sub execute_stacks { +# my $date = shift; +# my $pretend = shift; +# foreach my $target (@target_list) { +# foreach my $filter (@filter_list) { +# if (exists($extra_processing{$target})) { +# my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter); +# if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :( +# next; +# } +# if ($Nexposures == 0) { +# if ($debug == 1) { +# print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; +# } +# next; +# } +# else { +# do_extra_processing($date,$target,$filter,$pretend); +# } + +# } +# if ($science_config{$target}{STACKABLE} == 1) { +# my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter); +# if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :( +# if ($debug == 1) { +# print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; +# } +# $metadata_out{nsState} = 'FORCETOWARP'; +# next; +# } +# if ($Nexposures == 0) { +# if ($debug == 1) { +# print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; +# } +# next; +# } +# if ($Nalready != 0) { +# if ($debug == 1) { +# print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; +# } +# unless ($metadata_out{nsState} eq 'FORCETOWARP') { +# $metadata_out{nsState} = 'STACKING'; +# } +# next; +# } +# unless (($metadata_out{nsState} eq 'FORCETOWARP')||($metadata_out{nsState} eq 'STACKING')) { +# $metadata_out{nsState} = 'QUEUESTACKS'; +# } +# unless(defined($pretend)) { +# if ($debug == 1) { +# print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; +# } +# stack_queue($date,$target,$filter); +# } +# if (defined($pretend)) { +# add_to_macro_list('check_stacks',$stackable_list{$target},$date,$target,$filter); +# } +# else { +# add_to_macro_list('queue_stacks',$stackable_list{$target},$date,$target,$filter); +# } +# } +# else { +# # print STDERR "execute_stacks: Target $target is not auto-stackable.\n"; +# } +# } +# } +# } + +# sub post_stack_queue { +# my $date = shift; +# my $target = shift; + +# my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); +# # check warp stage == chip stage +# my $db = init_gpc_db(); + +# my $trunc_date = $date; $trunc_date =~ s/-//g; + +# my $where = " label = '$label' AND data_group = '$data_group' "; + +# my $stack_full_sth = "SELECT * from stackRun where $where "; +# my $stack_done_sth = "SELECT * from stackRun LEFT JOIN stackSumSkyfile USING(stack_id) WHERE $where "; +# $stack_done_sth .= " AND ((state = 'full') OR (state = 'new' && fault != 0)) "; +# if ($debug == 1) { +# print STDERR "post_stack_queue: database queries:\n"; +# print STDERR "$stack_full_sth\n"; +# print STDERR "$stack_done_sth\n"; +# } +# my $stack_full_ref = $db->selectall_arrayref( $stack_full_sth ); +# my $stack_done_ref = $db->selectall_arrayref( $stack_done_sth ); + +# return($#{ $stack_full_ref } + 1, $#{ $stack_done_ref } + 1); +# } + +# sub confirm_stacks { +# my $date = shift; +# my $pretend = shift; + +# foreach my $target (@target_list) { +# if ($stackable_list{$target} == 1) { +# my ($Nstacks, $Nattempted) = post_stack_queue($date,$target); +# if ($debug == 1) { +# print STDERR "confirm_stacks: Target $target on $date has $Nattempted attempts on $Nstacks.\n"; +# } +# if ($Nstacks != $Nattempted) { +# if ($debug == 1) { +# print STDERR "confirm_stacks: Target $target on $date is not done stacking. $Nstacks $Nattempted\n"; +# } +# if ($metadata_out{nsState} eq 'CONFIRM_STACKING') { +# $metadata_out{nsState} = 'STACKING'; +# } +# next; +# } +# if ($Nstacks == 0) { +# if ($debug == 1) { +# print STDERR "confirm_stacks: Target $target on $date has no stacks. Skipping.\n"; +# } +# next; +# } +# if ($metadata_out{nsState} eq 'CONFIRM_STACKING') { +# if (defined($pretend)) { +# add_to_macro_list('check_confirm_stacks',$stackable_list{$target},$date,$target); +# } +# else { +# add_to_macro_list('confirm_stacks',$stackable_list{$target},$date,$target); +# } +# } +# } +# } +# } + +# # +# # Extra processing +# ################################################################################ + +# sub verify_uniqueness_diff { +# my $warp_id_1 = shift; +# my $warp_id_2 = shift; +# my $date = shift; +# my $target = shift; + +# my $db = init_gpc_db(); +# $date =~ s/-//g; +# my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + +# my $sth = "SELECT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2"; +# my $data_ref = $db->selectall_arrayref( $sth ); + +# return($#{ $data_ref } + 1); +# } + +# sub do_extra_processing { +# my $date = shift; +# my $target = shift; +# my $filter = shift; +# my $pretend = shift; +# my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + +# if ($target eq 'OSS') { +# my $db = init_gpc_db(); + +# my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; +# $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object"; +# print STDERR "$obj_sth\n"; +# my $object_ref = $db->selectall_arrayref( $obj_sth ); + +# foreach my $object_row (@{ $object_ref }) { +# my $this_object = shift @{ $object_row }; +# my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; +# $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; +# $input_sth .= " ORDER BY dateobs "; + +# my $warps = $db->selectall_arrayref( $input_sth ); + +# if (($#{ $warps } + 1) % 2 != 0) { +# print STDERR "Number of input warps to make OSS diffs is not even! $#{ $warps }\n"; +# last; +# } + +# while ($#{ $warps } > -1) { +# my $input_warp = shift @{ $warps }; +# my $template_warp = shift @{ $warps }; +# my $input_exp_id = ${ $input_warp }[0]; +# my $template_exp_id = ${ $template_warp }[0]; + +# my $input_warp_id = ${ $input_warp }[1]; +# my $template_warp_id = ${ $template_warp }[1]; + +# if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { +# print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id\n"; +# next; +# } + +# my $cmd = "$difftool -dbname $dbname -definewarpwarp "; +# $cmd .= "-input_label $label -template_label $label "; +# $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. +# $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; +# $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# # $cmd .= " -pretend "; +# if (defined($pretend)) { +# $cmd .= ' -pretend '; +# } +# if ($debug == 1) { +# $cmd .= ' -pretend '; +# } +# print STDERR "EXTRA_PROCESSING: $cmd\n"; +# if (($debug == 0)&&(!defined($pretend))) { +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $cmd, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# } +# } +# } +# } +# } + + + + +# +# Auto-Clean +################################################################################ + +# sub construct_cleantool_args { +# my $date = shift; +# my $target = shift; +# my $mode = shift; + +# my $command = $clean_commands{$mode}; +# my $retention_time; +# if (exists($cleanmods_list{$target}{$mode})) { +# $retention_time = $cleanmods_list{$target}{$mode}; +# } +# else { +# $retention_time = $clean_retention{$mode}; +# } +# if ($retention_time <= 0) { +# return("no clean","true"); +# } + +# my ($year,$month,$day) = split /-/,$date; +# my $dt = DateTime->new(year => $year, month => $month, day => $day, +# hour => 0, minute => 0, second => 0, nanosecond => 0, +# time_zone => 'Pacific/Honolulu'); + +# $dt->subtract(days => $retention_time); +# my $cleaning_date = $dt->ymd; + +# my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); + +# $command =~ s/\@DBNAME\@/$dbname/; +# $command =~ s/\@LABEL\@/$label/; +# $command =~ s/\@WORKDIR\@/$workdir/; +# $command =~ s/\@OBS_MODE\@/$obs_mode/; +# $command =~ s/\@OBJECT\@/$object/; +# $command =~ s/\@TESS_ID\@/$tess_id/; +# $command =~ s/\@DIST_GROUP\@/$dist_group/; +# $command =~ s/\@DATA_GROUP\@/$data_group/; +# $command =~ s/\@REDUCTION\@/$reduction/; +# $command =~ s/\@CURRENT_DATE\@/$cleaning_date/; + +# if ($debug == 1) { +# $command .= ' -pretend '; +# } +# return($cleaning_date,$command); +# } + +# sub execute_cleans { +# my $date = shift; +# my $pretend = shift; + +# foreach my $mode (sort (keys %clean_config)) { +# foreach my $target (sort (keys %science_config)) { +# if (exists($noclean_list{$target})) { +# next; +# } +# my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); +# if ($cleaning_date eq 'no clean') { +# next; +# } +# print STDERR "$command\n"; +# if (!(defined($pretend) || $debug == 1)) { +# # print STDERR "BEAR IS DRIVING!?\n"; +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $command, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# add_to_macro_list('clean_old',1,$date,$target,$mode); +# } +# } +# } +# return(0); +# } + +# +# Utilities +################################################################################ + +sub get_observing_state { + my $date = shift; + my $eon_dt = DateTime->new( year => $datetime->year, + month => $datetime->month, + day => $datetime->day, + hour => 6, + minute => 30, + second => 0, + time_zone => 'Pacific/Honolulu'); + + foreach my $eon (keys %eon_config) { + my $command = "$regtool -processedexp -simple "; + $command .= " -dbname $dbname "; + $command .= " -dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; + $command .= " -object $eon_config{$eon}{OBJECT} " if defined($eon_config{$eon}{OBJECT}); + $command .= " -obs_mode $eon_config{$eon}{OBSMODE} " if defined($eon_config{$eon}{OBSMODE}); + $command .= " -exp_type $eon_config{$eon}{EXPTYPE} " if defined($eon_config{$eon}{EXPTYPE}); + $command .= " -comment $eon_config{$eon}{COMMENT} " if defined($eon_config{$eon}{COMMENT}); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + my @eon_exposures = split /\n/, (join '', @$stdout_buf); + if ($#eon_exposures >= 0) { + return("END_OF_NIGHT"); + } + } + + if (DateTime->compare($datetime,$eon_dt) < 1) { + return("OBSERVING"); + } + else { + return("END_OF_NIGHT"); + } +} + +sub get_tool_parameters { + my $date = shift; + my $target = shift; + my $workdir_date = $date; $workdir_date =~ s%-%/%g; + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $label = "${target}.nightlyscience"; + my $workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}"; + my $obs_mode = $science_config{$target}{OBS_MODE}; + my $object = $science_config{$target}{OBJECT}; + my $comment = $science_config{$target}{COMMENT}; + my $dist_group = $science_config{$target}{DISTRIBUTION}; + my $data_group = "${target}.${trunc_date}"; + my $tess_id = $science_config{$target}{TESS}; + my $reduction = $science_config{$target}{REDUCTION}; + return($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction); +} + +sub get_dettool_parameters { + my $date = shift; + my $target = shift; + my $workdir_date = $date; $workdir_date =~ s%-%/%g; + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $exp_type = $detrend_config{$target}{EXP_TYPE}; + my $det_type = $detrend_config{$target}{DET_TYPE}; + my $ref_det_id = $detrend_config{$target}{REF_DET_ID}; + my $ref_iter = $detrend_config{$target}{REF_ITER}; + my $maxN = $detrend_config{$target}{MAX_EXP}; + my $det_filter = $detrend_config{$target}{DET_FILTER}; + my $internal_filter; + if (defined($det_filter)) { + $internal_filter = $det_filter; $internal_filter =~ s/\..*//; + $internal_filter = '.' . $internal_filter; + } + else { + $internal_filter = ''; + } + my $lc_type = lc($exp_type); + my $label = "${lc_type}${internal_filter}.$trunc_date"; + my $workdir = 'neb://@HOST@.0/' . $dbname . "/detverify.nt/${workdir_date}/${lc_type}${internal_filter}"; + return($label,$workdir,$det_filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN); +} + +sub add_to_macro_list { + my $proc_mode = shift; + my $do_or_do_not = shift; + my $date = shift; + my $target = shift; + my $mode = shift; + + if ((defined($macro_config{$proc_mode}))&&($do_or_do_not)) { + unless (defined($metadata_out{N_MACROS})) { + $metadata_out{N_MACROS} = 0; + } + my $N = $metadata_out{N_MACROS}; + $metadata_out{"ns${N}Macro"} = $macro_config{$proc_mode}; + if ($debug == 1) { + print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_config{$proc_mode}\n"; + } + if (defined($date)&&(defined($target))) { + + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) + = get_tool_parameters($date,$target); + $metadata_out{"ns${N}Macro"} =~ s/\@LABEL\@/$label/; + $metadata_out{"ns${N}Macro"} =~ s/\@WORKDIR\@/$workdir/; + $metadata_out{"ns${N}Macro"} =~ s/\@OBS_MODE\@/$obs_mode/; + $metadata_out{"ns${N}Macro"} =~ s/\@OBJECT\@/$object/; + $metadata_out{"ns${N}Macro"} =~ s/\@COMMENT\@/$comment/; + $metadata_out{"ns${N}Macro"} =~ s/\@TESS_ID\@/$tess_id/; + $metadata_out{"ns${N}Macro"} =~ s/\@DIST_GROUP\@/$dist_group/; + $metadata_out{"ns${N}Macro"} =~ s/\@DATA_GROUP\@/$data_group/; + $metadata_out{"ns${N}Macro"} =~ s/\@REDUCTION\@/$reduction/; + } + if (defined($mode)) { + $metadata_out{"ns${N}Macro"} =~ s/\@EXTRA\@/$mode/; + } + if (defined($date)) { + $metadata_out{"ns${N}Macro"} =~ s/\@DATE\@/$date/; + } + if ($debug == 1) { + print STDERR "DONE WITH A MACRO: ns${N}Macro $proc_mode $macro_config{$proc_mode}\n"; + } + $metadata_out{N_MACROS} ++; + } +} + +sub init_gpc_db { + use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + my $dbserver = 'ippdb01'; + my $dbuser = 'ippuser'; + my $dbpass = 'ippuser'; + $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + return($db); +} + +sub return_metadata { + my $date = shift; + print STDOUT "autoStack MULTI\n\n"; + print STDOUT "autoStack METADATA\n"; + print STDOUT " date STR $date\n"; + foreach my $k (sort(keys %metadata_out)) { + print STDOUT " $k STR $metadata_out{$k}\n"; + } + print STDOUT "END\n"; +} + +sub my_die { + my $msg = shift; # Warning message on die + my $stage = shift; # stage name + my $stage_id = shift; # identifier + my $exit_code = shift; # Exit code + # outputImage and path_base are globals + + carp($msg); + exit $exit_code; +} + +sub my_trace { + if ($debug == 1) { + foreach my $thing (@_) { + carp($thing); + } + } +} Index: /branches/czw_branch/20101203/ippTasks/nightly_stacks.pro =================================================================== --- /branches/czw_branch/20101203/ippTasks/nightly_stacks.pro (revision 30013) +++ /branches/czw_branch/20101203/ippTasks/nightly_stacks.pro (revision 30014) @@ -7,9 +7,10 @@ ns.detrends.off ns.dqstats.on - ns.sweetspot.on - ns.registration.on - ns.burntool.on - ns.chips.on +# ns.sweetspot.on +# ns.registration.on +# ns.burntool.on +# ns.chips.on ns.stacks.on + ns.diffs.on end @@ -18,9 +19,10 @@ ns.detrends.off ns.dqstats.off - ns.sweetspot.off - ns.registration.off - ns.burntool.off - ns.chips.off +# ns.sweetspot.off +# ns.registration.off +# ns.burntool.off +# ns.chips.off ns.stacks.off + ns.diffs.off end @@ -51,33 +53,33 @@ end -macro ns.sweetspot.on - task ns.sweetspot.load - active true - end -end - -macro ns.registration.on - task ns.registration.load - active true - end -end - -macro ns.burntool.on - task ns.burntool.load - active true - end - task ns.burntool.run - active true - end -end - -macro ns.chips.on - task ns.chips.load - active true - end - task ns.chips.run - active true - end -end +# macro ns.sweetspot.on +# task ns.sweetspot.load +# active true +# end +# end + +# macro ns.registration.on +# task ns.registration.load +# active true +# end +# end + +# macro ns.burntool.on +# task ns.burntool.load +# active true +# end +# task ns.burntool.run +# active true +# end +# end + +# macro ns.chips.on +# task ns.chips.load +# active true +# end +# task ns.chips.run +# active true +# end +# end macro ns.stacks.on @@ -93,4 +95,13 @@ end +macro ns.diffs.on + task ns.diffs.load + active false + end + task ns.diffs.run + active true + end +end + macro ns.initday.off task ns.initday.load @@ -111,33 +122,33 @@ end -macro ns.sweetspot.off - task ns.sweetspot.load - active false - end -end - -macro ns.registration.off - task ns.registration.load - active false - end -end - -macro ns.burntool.off - task ns.burntool.load - active false - end - task ns.burntool.run - active false - end -end - -macro ns.chips.off - task ns.chips.load - active false - end - task ns.chips.run - active false - end -end +# macro ns.sweetspot.off +# task ns.sweetspot.load +# active false +# end +# end + +# macro ns.registration.off +# task ns.registration.load +# active false +# end +# end + +# macro ns.burntool.off +# task ns.burntool.load +# active false +# end +# task ns.burntool.run +# active false +# end +# end + +# macro ns.chips.off +# task ns.chips.load +# active false +# end +# task ns.chips.run +# active false +# end +# end macro ns.stacks.off @@ -150,16 +161,27 @@ end -$ns_regPAGE = 0 -$ns_burnPAGE = 0 -$ns_RburnPAGE = 0 -$ns_RburnCELL = 0 -$ns_chipPAGE = 0 -$ns_RchipPAGE = 0 +macro ns.diffs.off + task ns.diffs.load + active false + end + task ns.diffs.run + active false + end +end + +# $ns_regPAGE = 0 +# $ns_burnPAGE = 0 +# $ns_RburnPAGE = 0 +# $ns_RburnCELL = 0 +# $ns_chipPAGE = 0 +# $ns_RchipPAGE = 0 $ns_stackPAGE = 0 $ns_RstackPAGE = 0 $ns_CstackPAGE = 0 +$ns_diffPAGE = 0 +$ns_RdiffPAGE = 0 book init nsData -book init nsBurntool +# book init nsBurntool # # Macros to control the book. @@ -177,5 +199,5 @@ echo $date $state end - book npages nsBurntool -var Npages +# book npages nsBurntool -var Npages end @@ -198,5 +220,5 @@ periods -exec $LOADEXEC periods -timeout 30 - trange 16:30:00 17:30:00 -nmax 1 + trange 0:00:00 1:00:00 -nmax 1 npending 1 @@ -204,5 +226,5 @@ stdout $LOGDIR/ns.initday.log stderr $LOGDIR/ns.initday.log - $today = `date +%Y-%m-%d` + $today = `date -u +%Y-%m-%d` book newpage nsData $today book setword nsData $today nsState NEW @@ -242,5 +264,5 @@ stdout $LOGDIR/ns.detrends.log stdout $LOGDIR/ns.detrends.log - $today = `date +%Y-%m-%d` + $today = `date -u +%Y-%m-%d` command automate_stacks.pl --queue_detrends --date $today @@ -278,5 +300,5 @@ stdout $LOGDIR/ns.dqstats.log stderr $LOGDIR/ns.dqstats.log - $today = `date +%Y-%m-%d` + $today = `date -u +%Y-%m-%d` command automate_stacks.pl --queue_dqstats --date $today @@ -313,5 +335,5 @@ stdout $LOGDIR/ns.sweetspot.log stderr $LOGDIR/ns.sweetspot.log - $today = `date +%Y-%m-%d` + $today = `date -u +%Y-%m-%d` command automate_stacks.pl --queue_sweetspot --date $today @@ -337,192 +359,192 @@ # Check to see if registration is complete # -task ns.registration.load - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.registration.log - - book getpage nsData $ns_regPAGE -var date - book getword nsData $date nsState -var ns_STATE - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.registration.load: " $ns_regPAGE $date $ns_STATE $Npages - end - - $ns_regPAGE ++ - if ($ns_regPAGE >= $Npages) set ns_regPAGE = 0 - option $date - - if ("$ns_STATE" != "NEW") break - $run = automate_stacks.pl --check_registration --date $date - command $run - end - - # success - task.exit 0 -# convert 'stdout' to book format - book delpage nsData $options:0 - ipptool2book stdout nsData -uniq -key date - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout - showcommand timeout - end -end - -# -# Check to see if the chips have been properly burntooled and are ready for queueing -# -task ns.chips.load - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 300 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.chips.log - - book getpage nsData $ns_chipPAGE -var date - book getword nsData $date nsState -var ns_STATE - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.chips.load: " $ns_chipPAGE $date $ns_STATE $Npages - end - - $ns_chipPAGE ++ - if ($ns_chipPAGE >= $Npages) set ns_chipPAGE = 0 - option $date - - if (("$ns_STATE" != "REGISTERED")&&("$ns_STATE" != "BURNING")) break - $run = automate_stacks.pl --check_chips --date $date - - if ("$ns_STATE" == "BURNING") - $run = $run --isburning - end - command $run - end - # success - task.exit 0 -# convert 'stdout' to book format - book delpage nsData $options:0 - ipptool2book stdout nsData -uniq -key date - - # remove the burntool page if we're done with it. - book getword nsData $options:0 nsState -var ns_STATE - if ("$ns_STATE" == "QUEUECHIPS") - book delpage nsBurntool $options:0 - end - - book getword nsData $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsData $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout - showcommand timeout - end -end - -# -# Check to see if the chips have been properly burntooled and are ready for queueing -# -task ns.chips.run - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 120 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.chips.log - - book getpage nsData $ns_RchipPAGE -var date - book getword nsData $date nsState -var ns_STATE - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.chips.run: " $ns_RchipPAGE $date $ns_STATE $Npages - end +# task ns.registration.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 30 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.registration.log + +# book getpage nsData $ns_regPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.registration.load: " $ns_regPAGE $date $ns_STATE $Npages +# end + +# $ns_regPAGE ++ +# if ($ns_regPAGE >= $Npages) set ns_regPAGE = 0 +# option $date + +# if ("$ns_STATE" != "NEW") break +# $run = automate_stacks.pl --check_registration --date $date +# command $run +# end + +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsData $options:0 +# ipptool2book stdout nsData -uniq -key date + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Check to see if the chips have been properly burntooled and are ready for queueing +# # +# task ns.chips.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 300 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.chips.log + +# book getpage nsData $ns_chipPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.chips.load: " $ns_chipPAGE $date $ns_STATE $Npages +# end + +# $ns_chipPAGE ++ +# if ($ns_chipPAGE >= $Npages) set ns_chipPAGE = 0 +# option $date + +# if (("$ns_STATE" != "REGISTERED")&&("$ns_STATE" != "BURNING")) break +# $run = automate_stacks.pl --check_chips --date $date + +# if ("$ns_STATE" == "BURNING") +# $run = $run --isburning +# end +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsData $options:0 +# ipptool2book stdout nsData -uniq -key date + +# # remove the burntool page if we're done with it. +# book getword nsData $options:0 nsState -var ns_STATE +# if ("$ns_STATE" == "QUEUECHIPS") +# book delpage nsBurntool $options:0 +# end + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Check to see if the chips have been properly burntooled and are ready for queueing +# # +# task ns.chips.run +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 120 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.chips.log + +# book getpage nsData $ns_RchipPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.chips.run: " $ns_RchipPAGE $date $ns_STATE $Npages +# end - $ns_RchipPAGE ++ - if ($ns_RchipPAGE >= $Npages) set ns_RchipPAGE = 0 - option $date - - if ("$ns_STATE" != "QUEUECHIPS") break - $BURNTOOLING = 0 - $run = automate_stacks.pl --queue_chips --date $date - command $run - end - # success - task.exit 0 -# convert 'stdout' to book format - book delpage nsData $options:0 - ipptool2book stdout nsData -uniq -key date - - book getword nsData $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsData $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout - showcommand timeout - end -end +# $ns_RchipPAGE ++ +# if ($ns_RchipPAGE >= $Npages) set ns_RchipPAGE = 0 +# option $date + +# if ("$ns_STATE" != "QUEUECHIPS") break +# $BURNTOOLING = 0 +# $run = automate_stacks.pl --queue_chips --date $date +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsData $options:0 +# ipptool2book stdout nsData -uniq -key date + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end @@ -557,4 +579,5 @@ command $run end + # success task.exit 0 @@ -718,313 +741,313 @@ end -# -# Generate a list of date ranges and chips that need to be processed with burntool -# -task ns.burntool.load - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.burntool.log - - book getpage nsData $ns_burnPAGE -var date - book getword nsData $date nsState -var ns_STATE - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.burntool.load: " $ns_burnPAGE $date $ns_STATE $Npages - end - - $ns_burnPAGE ++ - if ($ns_burnPAGE >= $Npages) set ns_burnPAGE = 0 - option $date +# # +# # Generate a list of date ranges and chips that need to be processed with burntool +# # +# task ns.burntool.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 30 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.burntool.log + +# book getpage nsData $ns_burnPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.burntool.load: " $ns_burnPAGE $date $ns_STATE $Npages +# end + +# $ns_burnPAGE ++ +# if ($ns_burnPAGE >= $Npages) set ns_burnPAGE = 0 +# option $date - if ("$ns_STATE" != "NEEDSBURNING") break - - $run = automate_stacks.pl --define_burntool --date $date - command $run - end - # success - task.exit 0 -# convert 'stdout' to book format -# book delpage nsBurntool $options:0 - ipptool2book stdout nsBurntool -uniq -key date - book setword nsData $options:0 nsState "QUEUEBURNING" - - book getword nsData $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsData $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout - showcommand timeout - end -end - -# -# Magically run burntool on the data, based on the information stored in our book. -# -task ns.burntool.run - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 10800 - npending 30 - task.exec - stdout NULL - stderr $LOGDIR/ns.burntool.log - - book getpage nsData $ns_RburnPAGE -var date - book getword nsData $date nsState -var ns_STATE - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.burntool.run: " $ns_RburnPAGE $date $ns_STATE $Npages - end - - $ns_RburnPAGE ++ - if ($ns_RburnPAGE >= $Npages) set ns_RburnPAGE = 0 - if (("$ns_STATE" != "QUEUEBURNING")&&("$ns_STATE" != "BURNING")) break - $BURNTOOLING = 1 - # Find out where in the list of jobs we are - book getword nsBurntool $date btN -var btN - book getword nsBurntool $date btNCounter -var btNcounter - - if ($VERBOSE > 5) - echo "ns.burntool.run: Status: " $btNcounter $date - end - - if ("$ns_STATE" == "QUEUEBURNING") - $new_state = "QUEUEBURNING" - end - if ("$ns_STATE" == "BURNING") - $new_state = "BURNING" - end - if ($btNcounter + 1 > $btN) - $new_state = "BURNING" - end - if ($btNcounter > $btN) - $btNcounter = 0 - end - if ($VERBOSE > 5) - echo "ns.burntool.run: Status: " $btNcounter $new_state - end +# if ("$ns_STATE" != "NEEDSBURNING") break + +# $run = automate_stacks.pl --define_burntool --date $date +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# # book delpage nsBurntool $options:0 +# ipptool2book stdout nsBurntool -uniq -key date +# book setword nsData $options:0 nsState "QUEUEBURNING" + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Magically run burntool on the data, based on the information stored in our book. +# # +# task ns.burntool.run +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 10800 +# npending 30 +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.burntool.log + +# book getpage nsData $ns_RburnPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.burntool.run: " $ns_RburnPAGE $date $ns_STATE $Npages +# end + +# $ns_RburnPAGE ++ +# if ($ns_RburnPAGE >= $Npages) set ns_RburnPAGE = 0 +# if (("$ns_STATE" != "QUEUEBURNING")&&("$ns_STATE" != "BURNING")) break +# $BURNTOOLING = 1 +# # Find out where in the list of jobs we are +# book getword nsBurntool $date btN -var btN +# book getword nsBurntool $date btNCounter -var btNcounter + +# if ($VERBOSE > 5) +# echo "ns.burntool.run: Status: " $btNcounter $date +# end + +# if ("$ns_STATE" == "QUEUEBURNING") +# $new_state = "QUEUEBURNING" +# end +# if ("$ns_STATE" == "BURNING") +# $new_state = "BURNING" +# end +# if ($btNcounter + 1 > $btN) +# $new_state = "BURNING" +# end +# if ($btNcounter > $btN) +# $btNcounter = 0 +# end +# if ($VERBOSE > 5) +# echo "ns.burntool.run: Status: " $btNcounter $new_state +# end - # Increment the counter in the book for the next job. - $counter_update = $btNcounter + 1 - book setword nsBurntool $date btNCounter $counter_update -# book setword nsData $options:0 nsState $options:1 - book setword nsData $date nsState $new_state - - # Get the current status of this job, and skip if it doesn't need to process. - sprintf status_label "bt%dStatus" $btNcounter - book getword nsBurntool $date $status_label -var status - - if ($VERBOSE > 5) - echo "ns.burntool.run: Status: " $btNcounter $status $date $status_label - end - if (("$status" == "FINISHED")||("$status" == "RUN")) break +# # Increment the counter in the book for the next job. +# $counter_update = $btNcounter + 1 +# book setword nsBurntool $date btNCounter $counter_update +# # book setword nsData $options:0 nsState $options:1 +# book setword nsData $date nsState $new_state + +# # Get the current status of this job, and skip if it doesn't need to process. +# sprintf status_label "bt%dStatus" $btNcounter +# book getword nsBurntool $date $status_label -var status + +# if ($VERBOSE > 5) +# echo "ns.burntool.run: Status: " $btNcounter $status $date $status_label +# end +# if (("$status" == "FINISHED")||("$status" == "RUN")) break - # Continue loading information to process this job - sprintf start_date_label "bt%dBegin" $btNcounter - sprintf end_date_label "bt%dEnd" $btNcounter - sprintf class_count_label "bt%dClass" $btNcounter - - book getword nsBurntool $date $start_date_label -var start_date - book getword nsBurntool $date $end_date_label -var end_date - book getword nsBurntool $date $class_count_label -var chip_counter - - # Lookup class_id/host pairs - list word -split $hostmatch:$chip_counter - $class_id = $word:0 - $host = $word:1 - host $host -# set.host.for.camera GPC1 $class_id - $logfile = "/data/$host.0/burntool_logs/$class_id.$start_date.log" - - $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile +# # Continue loading information to process this job +# sprintf start_date_label "bt%dBegin" $btNcounter +# sprintf end_date_label "bt%dEnd" $btNcounter +# sprintf class_count_label "bt%dClass" $btNcounter + +# book getword nsBurntool $date $start_date_label -var start_date +# book getword nsBurntool $date $end_date_label -var end_date +# book getword nsBurntool $date $class_count_label -var chip_counter + +# # Lookup class_id/host pairs +# list word -split $hostmatch:$chip_counter +# $class_id = $word:0 +# $host = $word:1 +# host $host +# # set.host.for.camera GPC1 $class_id +# $logfile = "/data/$host.0/burntool_logs/$class_id.$start_date.log" + +# $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile - echo "ns.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run - echo "ns.burntool.run: " $date $btN $btNcounter $chip_counter $new_state - - book setword nsBurntool $date $status_label RUN - option $date $new_state $status_label - - command $run -# command /bin/true - end - # success - task.exit 0 -# convert 'stdout' to book format - # Set data state based on if we're queueing or waiting - # Set the job state for success. - book setword nsBurntool $options:0 $options:2 FINISHED - - book getword nsData $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsData $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - book setword nsBurntool $options:0 $options:2 FAIL - showcommand failure - end - task.exit crash - book setword nsBurntool $options:0 $options:2 FAIL - showcommand crash - end - #operation times out? - task.exit timeout - book setword nsBurntool $options:0 $options:2 FAIL - showcommand timeout - end -end - - -# -# This is all burntool related stuff. -# - -macro burntool - if ($0 != 3) - echo "USAGE: burntool (dateobs_begin) (dateobs_end)" - break - end - - for i 0 $hostmatch:n - job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile - end -end -macro loadhosts - for i 0 $allhosts:n - controller host add $allhosts:$i - end -end - -# sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time. - -list hostmatch - XY01 ipp014 - XY03 ipp038 - XY05 ipp023 - XY10 ipp039 - XY12 ipp024 - XY15 ipp040 - XY17 ipp020 - XY21 ipp041 - XY23 ipp042 - XY25 ipp043 - XY27 ipp028 - XY31 ipp044 - XY33 ipp029 - XY35 ipp045 - XY37 ipp030 - XY41 ipp046 - XY43 ipp031 - XY45 ipp047 - XY47 ipp032 - XY51 ipp048 - XY53 ipp033 - XY55 ipp049 - XY57 ipp034 - XY61 ipp050 - XY63 ipp035 - XY65 ipp051 - XY67 ipp036 - XY72 ipp052 - XY74 ipp015 - XY76 ipp053 - XY02 ipp014 - XY04 ipp038 - XY06 ipp023 - XY11 ipp039 - XY13 ipp024 - XY14 ipp040 - XY16 ipp020 - XY20 ipp041 - XY22 ipp042 - XY24 ipp043 - XY26 ipp028 - XY30 ipp044 - XY32 ipp029 - XY34 ipp045 - XY36 ipp030 - XY40 ipp046 - XY42 ipp031 - XY44 ipp047 - XY46 ipp032 - XY50 ipp048 - XY52 ipp033 - XY54 ipp049 - XY56 ipp034 - XY60 ipp050 - XY62 ipp035 - XY64 ipp051 - XY66 ipp036 - XY71 ipp052 - XY73 ipp015 - XY75 ipp053 -end - -list allhosts - ipp043 - ipp014 - ipp015 - ipp023 - ipp024 - ipp020 - ipp028 - ipp029 - ipp030 - ipp031 - ipp032 - ipp033 - ipp034 - ipp035 - ipp036 - ipp038 - ipp039 - ipp040 - ipp041 - ipp042 - ipp044 - ipp045 - ipp046 - ipp047 - ipp048 - ipp049 - ipp050 - ipp051 - ipp052 - ipp053 -end +# echo "ns.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run +# echo "ns.burntool.run: " $date $btN $btNcounter $chip_counter $new_state + +# book setword nsBurntool $date $status_label RUN +# option $date $new_state $status_label + +# command $run +# # command /bin/true +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# # Set data state based on if we're queueing or waiting +# # Set the job state for success. +# book setword nsBurntool $options:0 $options:2 FINISHED + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# book setword nsBurntool $options:0 $options:2 FAIL +# showcommand failure +# end +# task.exit crash +# book setword nsBurntool $options:0 $options:2 FAIL +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# book setword nsBurntool $options:0 $options:2 FAIL +# showcommand timeout +# end +# end + + +# # +# # This is all burntool related stuff. +# # + +# macro burntool +# if ($0 != 3) +# echo "USAGE: burntool (dateobs_begin) (dateobs_end)" +# break +# end + +# for i 0 $hostmatch:n +# job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile +# end +# end +# macro loadhosts +# for i 0 $allhosts:n +# controller host add $allhosts:$i +# end +# end + +# # sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time. + +# list hostmatch +# XY01 ipp014 +# XY03 ipp038 +# XY05 ipp023 +# XY10 ipp039 +# XY12 ipp024 +# XY15 ipp040 +# XY17 ipp020 +# XY21 ipp041 +# XY23 ipp042 +# XY25 ipp043 +# XY27 ipp028 +# XY31 ipp044 +# XY33 ipp029 +# XY35 ipp045 +# XY37 ipp030 +# XY41 ipp046 +# XY43 ipp031 +# XY45 ipp047 +# XY47 ipp032 +# XY51 ipp048 +# XY53 ipp033 +# XY55 ipp049 +# XY57 ipp034 +# XY61 ipp050 +# XY63 ipp035 +# XY65 ipp051 +# XY67 ipp036 +# XY72 ipp052 +# XY74 ipp015 +# XY76 ipp053 +# XY02 ipp014 +# XY04 ipp038 +# XY06 ipp023 +# XY11 ipp039 +# XY13 ipp024 +# XY14 ipp040 +# XY16 ipp020 +# XY20 ipp041 +# XY22 ipp042 +# XY24 ipp043 +# XY26 ipp028 +# XY30 ipp044 +# XY32 ipp029 +# XY34 ipp045 +# XY36 ipp030 +# XY40 ipp046 +# XY42 ipp031 +# XY44 ipp047 +# XY46 ipp032 +# XY50 ipp048 +# XY52 ipp033 +# XY54 ipp049 +# XY56 ipp034 +# XY60 ipp050 +# XY62 ipp035 +# XY64 ipp051 +# XY66 ipp036 +# XY71 ipp052 +# XY73 ipp015 +# XY75 ipp053 +# end + +# list allhosts +# ipp043 +# ipp014 +# ipp015 +# ipp023 +# ipp024 +# ipp020 +# ipp028 +# ipp029 +# ipp030 +# ipp031 +# ipp032 +# ipp033 +# ipp034 +# ipp035 +# ipp036 +# ipp038 +# ipp039 +# ipp040 +# ipp041 +# ipp042 +# ipp044 +# ipp045 +# ipp046 +# ipp047 +# ipp048 +# ipp049 +# ipp050 +# ipp051 +# ipp052 +# ipp053 +# end Index: /branches/czw_branch/20101203/ippTools/src/regtool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/regtool.c (revision 30013) +++ /branches/czw_branch/20101203/ippTools/src/regtool.c (revision 30014) @@ -1386,5 +1386,7 @@ PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); - PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_STR(config->args, where, "-object", "object", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); Index: /branches/czw_branch/20101203/ippTools/src/regtoolConfig.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/regtoolConfig.c (revision 30013) +++ /branches/czw_branch/20101203/ippTools/src/regtoolConfig.c (revision 30014) @@ -313,4 +313,7 @@ psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by exposure obs_mode", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-comment", 0, "search by exposure comment", NULL); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); Index: /branches/czw_branch/20101203/ippconfig/recipes/nightly_science.config =================================================================== --- /branches/czw_branch/20101203/ippconfig/recipes/nightly_science.config (revision 30013) +++ /branches/czw_branch/20101203/ippconfig/recipes/nightly_science.config (revision 30014) @@ -1,32 +1,36 @@ -RETENTION_TIME U16 9000 - CLEAN_MODES MULTI CLEAN_MODES METADATA - MODE STR CHIP - COMMAND STR chiptool + MODE STR CHIP + COMMAND STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME U16 30 END CLEAN_MODES METADATA - MODE STR WARP - COMMAND STR warptool + MODE STR WARP + COMMAND STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME U16 7 END CLEAN_MODES METADATA - MODE STR DIFF - COMMAND STR difftool + MODE STR DIFF + COMMAND STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME S16 30 -# RETENTION_TIME S16 -1 -END -CLEAN_MODES METADATA - MODE STR DIST - COMMAND STR disttool +END +CLEAN_MODES METADATA + MODE STR DIST + COMMAND STR disttool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -label @LABEL@ -time_stamp_end @CURRENT_DATE@ RETENTION_TIME S16 10 - ALTERNATE_CMD STR A -END -CLEAN_MODES METADATA - MODE STR MAGICDS - COMMAND STR magicdstool +END +CLEAN_MODES METADATA + MODE STR MAGICDS + COMMAND STR magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ RETENTION_TIME S16 1 - ALTERNATE_CMD STR B +END + +END_OF_NIGHT MULTI +END_OF_NIGHT METADATA + NAME STR MORNING_DARKS + OBSMODE STR ENGINEERING + EXPTYPE STR DARK + OBJECT STR ENGINEERING + COMMENT STR darks_morn% END @@ -43,8 +47,4 @@ MACRO_DEFINITIONS MULTI -#MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_chips -# MACRO STR echo survey.del.destreak @LABEL@ -#END MACRO_DEFINITIONS METADATA PROC_MODE STR queue_chips @@ -55,360 +55,172 @@ MACRO STR survey.add.destreak @LABEL@ /data/ipp053.0/gpc1_destreak END -#MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_confirm_stacks -# MACRO STR echo survey.add.destreak @LABEL@ /data/ipp053.0/gpc1_destreak -#END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR clean_old -# MACRO STR + +TARGETS MULTI +TARGETS METADATA + NAME STR M31 + TESS STR M31 + OBSMODE STR M31 + OBJECT STR M31% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD01 + TESS STR MD01.V2 + OBSMODE STR MD + OBJECT STR MD01% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD02 + TESS STR MD02 + OBSMODE STR MD + OBJECT STR MD02% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD03 + TESS STR MD03 + OBSMODE STR MD + OBJECT STR MD03% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD04 + TESS STR MD04 + OBSMODE STR MD + OBJECT STR MD04% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD05 + TESS STR MD05 + OBSMODE STR MD + OBJECT STR MD05% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD06 + TESS STR MD06 + OBSMODE STR MD + OBJECT STR MD06% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD07 + TESS STR MD07 + OBSMODE STR MD + OBJECT STR MD07% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD08 + TESS STR MD08.V2 + OBSMODE STR MD + OBJECT STR MD08% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD09 + TESS STR MD09.V2 + OBSMODE STR MD + OBJECT STR MD09% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD10 + TESS STR MD10.V2 + OBSMODE STR MD + OBJECT STR MD10% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD11 + TESS STR MD11.V2 + OBSMODE STR MD + OBJECT STR MD11% + STACKABLE BOOL TRUE + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR STS + TESS STR STS + OBSMODE STR STS% + OBJECT STR STS% + STACKABLE BOOL FALSE + DIFFABLE BOOL FALSE +END +TARGETS METADATA + NAME STR SweetSpot + TESS STR RINGS.V0 + OBSMODE STR SS + STACKABLE BOOL FALSE + DIFFABLE BOOL TRUE + WARP S16 60 +END +TARGETS METADATA + NAME STR OSS + DISTRIBUTION STR SweetSpot + TESS STR RINGS.V0 + OBSMODE STR OSS + STACKABLE BOOL FALSE + DIFFABLE BOOL TRUE +END +TARGETS METADATA + NAME STR CNP + DISTRIBUTION STR CNP + TESS STR RINGS.V0 + OBSMODE STR CNP + STACKABLE BOOL FALSE + DIFFABLE BOOL FALSE +END +TARGETS METADATA + NAME STR ThreePi + TESS STR RINGS.V0 + OBSMODE STR 3PI + STACKABLE BOOL FALSE + DIFFABLE BOOL FALSE + CHIP S16 14 + DIFF S16 14 +END +# This is a hack, and I freely admit it. +# Removed the rest because we do not regularly run microtests. +# TARGETS METADATA +# NAME STR microtestMD01 +# TESS STR MD01 +# OBJECT STR MD01% +# COMMENT STR Microtest Obs% +# STACKABLE BOOL TRUE # END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_registration -# MACRO STR +# TARGETS METADATA +# NAME STR microtestMD01.noPattern +# TESS STR MD01 +# OBJECT STR MD01% +# COMMENT STR Microtest Obs% +# REDUCTION STR MICROTEST_NOPATTERN +# STACKABLE BOOL TRUE # END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR define_burntool -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR queue_burntool -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_stacks -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR queue_stacks -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_sweetspot -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR queue_sweetspot -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_dqstats -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR queue_dqstats -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_detrends -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR queue_detrends -# MACRO STR -# END - -TARGETS MULTI -TARGETS METADATA - NAME STR M31 - TESS STR M31 - OBSMODE STR M31 - OBJECT STR M31% -# COMMENT STR M31% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD01 - TESS STR MD01.V2 - OBSMODE STR MD - OBJECT STR MD01% -# COMMENT STR %MD01% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD02 - TESS STR MD02 - OBSMODE STR MD - OBJECT STR MD02% -# COMMENT STR %MD02% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD03 - TESS STR MD03 - OBSMODE STR MD - OBJECT STR MD03% -# COMMENT STR %MD03% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD04 - TESS STR MD04 - OBSMODE STR MD - OBJECT STR MD04% -# COMMENT STR %MD04% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD05 - TESS STR MD05 - OBSMODE STR MD - OBJECT STR MD05% -# COMMENT STR %MD05% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD06 - TESS STR MD06 - OBSMODE STR MD - OBJECT STR MD06% -# COMMENT STR %MD06% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD07 - TESS STR MD07 - OBSMODE STR MD - OBJECT STR MD07% -# COMMENT STR %MD07% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD08 - TESS STR MD08.V2 - OBSMODE STR MD - OBJECT STR MD08% -# COMMENT STR %MD08% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD09 - TESS STR MD09.V2 - OBSMODE STR MD - OBJECT STR MD09% -# COMMENT STR %MD09% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD10 - TESS STR MD10.V2 - OBSMODE STR MD - OBJECT STR MD10% -# COMMENT STR %MD10% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD11 - TESS STR MD11.V2 - OBSMODE STR MD - OBJECT STR MD11% -# COMMENT STR %MD11% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR STS - TESS STR STS - OBSMODE STR STS% - OBJECT STR STS% -# COMMENT STR Stellar Transit% - STACKABLE BOOL FALSE -END -TARGETS METADATA - NAME STR SweetSpot - TESS STR RINGS.V0 - OBSMODE STR SS -# OBJECT STR SS - STACKABLE BOOL FALSE - EXTRA_PROCESSING BOOL TRUE -# NOCLEAN BOOL TRUE - WARP S16 60 -END -TARGETS METADATA - NAME STR OSS - DISTRIBUTION STR SweetSpot - TESS STR RINGS.V0 - OBSMODE STR OSS - STACKABLE BOOL FALSE - EXTRA_PROCESSING BOOL TRUE -END -TARGETS METADATA - NAME STR CNP - DISTRIBUTION STR CNP - TESS STR RINGS.V0 - OBSMODE STR CNP - STACKABLE BOOL FALSE - EXTRA_PROCESSING BOOL FALSE -END -TARGETS METADATA - NAME STR ThreePi - TESS STR RINGS.V0 - OBSMODE STR 3PI -# OBJECT STR 3PI - STACKABLE BOOL FALSE - CHIP S16 14 - DIFF S16 14 -END -# This is a hack, and I freely admit it. -TARGETS METADATA - NAME STR microtestMD01 - TESS STR MD01 - OBJECT STR MD01% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD01.noPattern - TESS STR MD01 - OBJECT STR MD01% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD02 - TESS STR MD02 - OBJECT STR MD02% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD02.noPattern - TESS STR MD02 - OBJECT STR MD02% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD03 - TESS STR MD03 - OBJECT STR MD03% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD03.noPattern - TESS STR MD03 - OBJECT STR MD03% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD04 - TESS STR MD04 - OBJECT STR MD04% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD04.noPattern - TESS STR MD04 - OBJECT STR MD04% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD05 - TESS STR MD05 - OBJECT STR MD05% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD05.noPattern - TESS STR MD05 - OBJECT STR MD05% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD06 - TESS STR MD06 - OBJECT STR MD06% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD06.noPattern - TESS STR MD06 - OBJECT STR MD06% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD07 - TESS STR MD07 - OBJECT STR MD07% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD07.noPattern - TESS STR MD07 - OBJECT STR MD07% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD08 - TESS STR MD08 - OBJECT STR MD08% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD08.noPattern - TESS STR MD08 - OBJECT STR MD08% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD09 - TESS STR MD09 - OBJECT STR MD09% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD09.noPattern - TESS STR MD09 - OBJECT STR MD09% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD10 - TESS STR MD10 - OBJECT STR MD10% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD10.noPattern - TESS STR MD10 - OBJECT STR MD10% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END # Detrend verification data @@ -420,5 +232,4 @@ REF_ID S64 299 REF_ITER S32 0 -# FILTER STR MAX S32 10 END