Index: branches/pap/ippScripts/scripts/addstar_run.pl =================================================================== --- branches/pap/ippScripts/scripts/addstar_run.pl (revision 27708) +++ branches/pap/ippScripts/scripts/addstar_run.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: branches/pap/ippScripts/scripts/automate_stacks.pl =================================================================== --- branches/pap/ippScripts/scripts/automate_stacks.pl (revision 27708) +++ branches/pap/ippScripts/scripts/automate_stacks.pl (revision 28003) @@ -56,5 +56,5 @@ # Grab options my ( $date, $camera, $dbname, $logfile, $verbose, $manual); -my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only); +my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $check_mode); my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old); @@ -67,6 +67,8 @@ 'logfile=s' => \$logfile, 'verbose' => \$verbose, + 'debug' => \$debug, 'isburning' => \$isburning, 'force_stack_count' => \$force_stack_count, + 'check' => \$check_mode, 'test_mode' => \$test_mode, 'this_target_only=s' => \$this_target_only, @@ -115,5 +117,5 @@ defined $queue_chips or defined $queue_stacks or defined $check_chips or defined $check_stacks or - defined $test_mode or defined $clean_old; + defined $test_mode or defined $clean_old or defined $check_mode; # Configurable parameters from our config file. @@ -126,6 +128,7 @@ my %stackable_list = (); my %reduction_class = (); -my $retention_time = 9000; - +my %clean_commands = (); +my %clean_retention = (); +my %noclean_list = (); # Grab the configuration data. my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; @@ -140,7 +143,22 @@ my $metadata = $mdcParser->parse(join "", @$stdout_buf); foreach my $entry (@{ $metadata }) { - if (${ $entry }{name} eq 'RETENTION_TIME') { - $retention_time = ${ $entry }{value}; - } + if (@{ $entry }{name} eq 'CLEAN_MODES') { + my @mode_data = @{ ${ $entry }{value} }; + my $this_mode = ''; + foreach my $mentry (@mode_data) { + if (${ $mentry }{name} eq 'MODE') { + $this_mode = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'COMMAND') { + $clean_commands{$this_mode} = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'RETENTION_TIME') { + $clean_retention{$this_mode} = ${ $mentry }{value}; + } + } + } +# if (${ $entry }{name} eq 'RETENTION_TIME') { +# $retention_time = ${ $entry }{value}; +# } elsif (${ $entry }{name} eq 'FILTERS') { push @filter_list, ${ $entry }{value}; @@ -172,4 +190,7 @@ $reduction_class{$this_target} = ${ $tentry }{value}; } + elsif (${ $tentry }{name} eq 'NOCLEAN') { + $noclean_list{$this_target} = ${ $tentry }{value}; + } } } @@ -217,11 +238,22 @@ if (defined($test_mode)) { $debug = 1; -} - -if (defined($check_registration) || defined($test_mode)) { + my $z; + foreach $z (@target_list) { + print "TARGET: $z $tessID_list{$z} $obsmode_list{$z} $object_list{$z} $comment_list{$z} $stackable_list{$z} $reduction_class{$z}\n"; + } + foreach $z (@filter_list) { + print "FILTER: $z\n"; + } + foreach $z (keys (%clean_commands)) { + print "CLEAN: $z $clean_commands{$z} $clean_retention{$z}\n"; + } + +} + +if (defined($check_registration) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'NEW'; my ($Nsummit_exp,$Nfaults) = check_summit_copy($date); return_metadata($date); - unless (defined($test_mode)) { exit(0); } + unless (defined($test_mode) || defined($check_mode)) { exit(0); } } if (defined($define_burntool) || defined($test_mode)) { @@ -236,5 +268,5 @@ unless (defined($test_mode)) { exit(0); } } -if (defined($check_chips) || defined($test_mode)) { +if (defined($check_chips) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'QUEUECHIPS'; &execute_chips($date,"pretend"); @@ -243,5 +275,5 @@ } return_metadata($date); - unless (defined($test_mode)) { exit(0); } + unless (defined($test_mode) || defined($check_mode)) { exit(0); } } if (defined($queue_chips)) { @@ -251,5 +283,5 @@ exit(0); } -if (defined($check_stacks) || defined($test_mode)) { +if (defined($check_stacks) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'TOWARP'; &execute_stacks($date,"pretend"); @@ -258,9 +290,12 @@ } return_metadata($date); - unless (defined($test_mode)) { exit(0); } + unless (defined($test_mode) || defined($check_mode)) { exit(0); } } if (defined($queue_stacks)) { $metadata_out{nsState} = 'STACKING'; &execute_stacks($date); + if ($metadata_out{nsState} eq 'QUEUESTACKING') { + $metadata_out{nsState} = 'STACKING_POSSIBLE_ERROR'; + } return_metadata($date); exit(0); @@ -420,8 +455,8 @@ } - if ($#{ $stderr_buf } > -1) { - $metadata_out{nsState} = 'ERROR'; - return(1); - } +# if ($#{ $stderr_buf } > -1) { +# $metadata_out{nsState} = 'ERROR'; +# return(1); +# } my $N = 0; @@ -429,6 +464,9 @@ foreach my $row (@burntool_entries) { my ($trash,$start,$end); - ($trash,$trash,$start,$end) = (split /\s+/,$row); - + (undef,$trash,$start,$end) = (split /\s+/,$row); + if ($trash ne 'burntool') { + $metadata_out{nsState} = 'ERROR'; + return(1); + } for (my $class_counter = 0; $class_counter < 60; $class_counter++) { $metadata_out{"bt${N}Begin"} = $start; @@ -650,5 +688,5 @@ my @input_exposures = split /\n/, (join '', @$stdout_buf); foreach my $entry (@input_exposures) { - my ($skycell, @trash) = split /\s+/, $entry; + my ($warp_tess_id,$skycell, @trash) = split /\s+/, $entry; @trash = verify_uniqueness_stack($skycell,$date,$target,$filter); $Nalready += $trash[0]; @@ -681,21 +719,32 @@ my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter); if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :( - # print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; + if ($debug == 1) { + print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; + } $metadata_out{nsState} = 'FORCETOWARP'; next; } if ($Nexposures == 0) { - # print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; + if ($debug == 1) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; + } next; } if ($Nalready != 0) { - # print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; + if ($debug == 1) { + print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; + } + unless ($metadata_out{nsState} eq 'FORCETOWARP') { + $metadata_out{nsState} = 'STACKING'; + } next; } - unless ($metadata_out{nsState} eq 'FORCETOWARP') { + unless (($metadata_out{nsState} eq 'FORCETOWARP')||($metadata_out{nsState} eq 'STACKING')) { $metadata_out{nsState} = 'QUEUESTACKS'; } unless(defined($pretend)) { - # print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; + if ($debug == 1) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; + } stack_queue($date,$target,$filter); } @@ -716,4 +765,8 @@ my $date = shift; my $target = shift; + my $mode = shift; + + my $command = $clean_commands{$mode}; + my $retention_time = $clean_retention{$mode}; my ($year,$month,$day) = split /-/,$date; @@ -725,6 +778,6 @@ my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); - - my $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -data_group $data_group "; + my $args = $command; + $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; if ($debug == 1) { $args .= ' -pretend '; @@ -737,29 +790,22 @@ my $pretend = shift; - foreach my $target (@target_list) { - my ($cleaning_date,$args) = construct_cleantool_args($date,$target); - - my $command = $chiptool . $args; - print STDERR "$command\n"; - if (!(defined($pretend) || $debug == 1)) { + foreach my $mode (keys (%clean_commands)) { + foreach my $target (@target_list) { + if (exists($noclean_list{$target})) { + next; + } + my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); + + print STDERR "$command\n"; + if (!(defined($pretend) || $debug == 1)) { # print STDERR "BEAR IS DRIVING!?\n"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $command, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - } - $command = $warptool . $args; - print STDERR "$command\n"; - if (!(defined($pretend) || $debug == 1)) { -# print STDERR "BEAR IS DRIVING $pretend $debug!?\n"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $command, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + } + } } return(0); Index: branches/pap/ippScripts/scripts/calibrate_dvo.pl =================================================================== --- branches/pap/ippScripts/scripts/calibrate_dvo.pl (revision 27708) +++ branches/pap/ippScripts/scripts/calibrate_dvo.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/camera_exp.pl =================================================================== --- branches/pap/ippScripts/scripts/camera_exp.pl (revision 27708) +++ branches/pap/ippScripts/scripts/camera_exp.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: branches/pap/ippScripts/scripts/chip_imfile.pl =================================================================== --- branches/pap/ippScripts/scripts/chip_imfile.pl (revision 27708) +++ branches/pap/ippScripts/scripts/chip_imfile.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: branches/pap/ippScripts/scripts/detrend_correct_imfile.pl =================================================================== --- branches/pap/ippScripts/scripts/detrend_correct_imfile.pl (revision 27708) +++ branches/pap/ippScripts/scripts/detrend_correct_imfile.pl (revision 28003) @@ -8,7 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; -print "command: @ARGV\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/detrend_norm_apply.pl =================================================================== --- branches/pap/ippScripts/scripts/detrend_norm_apply.pl (revision 27708) +++ branches/pap/ippScripts/scripts/detrend_norm_apply.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/detrend_norm_calc.pl =================================================================== --- branches/pap/ippScripts/scripts/detrend_norm_calc.pl (revision 27708) +++ branches/pap/ippScripts/scripts/detrend_norm_calc.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -56,5 +57,5 @@ # force det_type to be upper-case in this script -my $det_type = uc($det_type); +$det_type = uc($det_type); my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $det_id, $iter, $PS_EXIT_CONFIG_ERROR ); # IPP configuration Index: branches/pap/ippScripts/scripts/detrend_norm_exp.pl =================================================================== --- branches/pap/ippScripts/scripts/detrend_norm_exp.pl (revision 27708) +++ branches/pap/ippScripts/scripts/detrend_norm_exp.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/detrend_process_exp.pl =================================================================== --- branches/pap/ippScripts/scripts/detrend_process_exp.pl (revision 27708) +++ branches/pap/ippScripts/scripts/detrend_process_exp.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/detrend_process_imfile.pl =================================================================== --- branches/pap/ippScripts/scripts/detrend_process_imfile.pl (revision 27708) +++ branches/pap/ippScripts/scripts/detrend_process_imfile.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/detrend_reject_exp.pl =================================================================== --- branches/pap/ippScripts/scripts/detrend_reject_exp.pl (revision 27708) +++ branches/pap/ippScripts/scripts/detrend_reject_exp.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/detrend_resid_imfile.pl =================================================================== --- branches/pap/ippScripts/scripts/detrend_resid_imfile.pl (revision 27708) +++ branches/pap/ippScripts/scripts/detrend_resid_imfile.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/detrend_stack.pl =================================================================== --- branches/pap/ippScripts/scripts/detrend_stack.pl (revision 27708) +++ branches/pap/ippScripts/scripts/detrend_stack.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/diff_skycell.pl =================================================================== --- branches/pap/ippScripts/scripts/diff_skycell.pl (revision 27708) +++ branches/pap/ippScripts/scripts/diff_skycell.pl (revision 28003) @@ -7,6 +7,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; @@ -398,8 +399,8 @@ $command .= " -hostname $host" if defined $host; $command .= " -path_base $outroot" if defined $outroot; - $command .= " -dbname $dbname" if defined $dbname; } else { $command .= " -updatediffskyfile"; } + $command .= " -dbname $dbname" if defined $dbname; run(command => $command, verbose => $verbose); } Index: branches/pap/ippScripts/scripts/dist_advancerun.pl =================================================================== --- branches/pap/ippScripts/scripts/dist_advancerun.pl (revision 27708) +++ branches/pap/ippScripts/scripts/dist_advancerun.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -174,10 +175,10 @@ } &my_die("unable to find path", $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path; - my $component_dir = find_componentdir($destdir, $path); + my $component_dir = find_componentdir($destdir, $path); # print MANIFEST "$component METADATA\n"; print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n"; } print MANIFEST "END\n\n"; - close MANIFEST or + close MANIFEST or &my_die("Unable to close dirinfo file $dirinfo", $dist_id, $PS_EXIT_UNKNOWN_ERROR); } Index: branches/pap/ippScripts/scripts/dist_bundle.pl =================================================================== --- branches/pap/ippScripts/scripts/dist_bundle.pl (revision 27708) +++ branches/pap/ippScripts/scripts/dist_bundle.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; # print "\n\n"; -# print "Starting script $0 on $host\n\n"; +# print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -115,5 +116,5 @@ # Get the list of data products for this component -# note: We my_die in get_file_list if something goes wrong. +# note: We my_die in get_file_list if something goes wrong. my $file_list = get_file_list($stage, $component, $path_base, $clean); @@ -193,5 +194,5 @@ $num_files++; if ($image_type && $nan_masked_pixels) { - # save the + # save the if ($image_type eq 'image') { $image = $file_name; @@ -222,5 +223,5 @@ # One last check as to whether magic has been applied to the inputs - # Note: the sql for disttool -pendingcomponent won't select a component that + # Note: the sql for disttool -pendingcomponent won't select a component that # requires magic and hasn't been magicked, but we check again here @@ -327,5 +328,5 @@ ### Pau. -# return the image type (image, mask, or variance) if this file rule refers to +# return the image type (image, mask, or variance) if this file rule refers to # one of the big fits files that are not included in a clean distribution sub get_image_type { @@ -360,5 +361,5 @@ sub open_with_retries { my $name = shift; - + my $tries = 1; my $max_tries = 5; @@ -372,5 +373,5 @@ } } - + &my_die("failed to open $name after $max_tries tries\n", $component, $PS_EXIT_DATA_ERROR) if (!$opened); Index: branches/pap/ippScripts/scripts/dist_cleanup.pl =================================================================== --- branches/pap/ippScripts/scripts/dist_cleanup.pl (revision 27708) +++ branches/pap/ippScripts/scripts/dist_cleanup.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/dist_component.pl =================================================================== --- branches/pap/ippScripts/scripts/dist_component.pl (revision 27708) +++ branches/pap/ippScripts/scripts/dist_component.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/dist_defineruns.pl =================================================================== --- branches/pap/ippScripts/scripts/dist_defineruns.pl (revision 27708) +++ branches/pap/ippScripts/scripts/dist_defineruns.pl (revision 28003) @@ -11,6 +11,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- branches/pap/ippScripts/scripts/dist_make_fileset.pl (revision 27708) +++ branches/pap/ippScripts/scripts/dist_make_fileset.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -46,5 +47,5 @@ 'dist_id=s' => \$dist_id, # distribution run identifier 'dist_dir=s' => \$dist_dir, # directory containing dist run outputs - 'target_id=s' => \$target_id, # + 'target_id=s' => \$target_id, # 'stage=s' => \$stage, # raw, chip, camera, fake, warp, stack, or diff 'stage_id=s' => \$stage_id, # exp_id, chip_id, etc. @@ -209,5 +210,5 @@ my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; - my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) or die "Cannot connect to database.\n"; @@ -227,10 +228,10 @@ # - # we are a long ways away from the rawExp in the pipeline. Rather than do some + # we are a long ways away from the rawExp in the pipeline. Rather than do some # very long joins in disttool, we look up the exp_name in the database using DBI # my $dbh = getDBHandle($ipprc, $dbname); - my $query; + my $query; if ($stage eq 'raw') { Index: branches/pap/ippScripts/scripts/dqstats_bundle.pl =================================================================== --- branches/pap/ippScripts/scripts/dqstats_bundle.pl (revision 27708) +++ branches/pap/ippScripts/scripts/dqstats_bundle.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; @@ -37,30 +38,30 @@ my ( $dqstats_id, $camera, $uri, $dbname, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( - 'dqstats_id=s' => \$dqstats_id, # dqstatsrun ID - 'camera|c=s' => \$camera, # camera - 'dbname|d=s' => \$dbname, # database name - 'uri=s' => \$uri, # output file destination - 'verbose' => \$verbose, # print to stdout - 'no-update' => \$no_update, # Update the database - 'no-op' => \$no_op, # don't do the operations - 'redirect-output' => \$redirect, - 'save-temps' => \$save_temps, # save temporary files - ) or pod2usage( 2 ); + 'dqstats_id=s' => \$dqstats_id, # dqstatsrun ID + 'camera|c=s' => \$camera, # camera + 'dbname|d=s' => \$dbname, # database name + 'uri=s' => \$uri, # output file destination + 'verbose' => \$verbose, # print to stdout + 'no-update' => \$no_update, # Update the database + 'no-op' => \$no_op, # don't do the operations + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # save temporary files + ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( -msg => "Required options --dqstats_id --camera", - -exitval => 3, - ) unless + -exitval => 3, + ) unless defined $dqstats_id and defined $camera; -my $ipprc = PS::IPP::Config->new( $camera ) +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $dqstats_id, $PS_EXIT_CONFIG_ERROR ); # IPP config -#my $logDest = $ipprc->filename("LOG.EXP", $uri) +#my $logDest = $ipprc->filename("LOG.EXP", $uri) # or my_die("Missing entry from camera config", $dqstats_id, $PS_EXIT_CONFIG_ERROR); if ($redirect) { -# $ipprc->redirect_output($logDest) -# or my_die( "Unable to redirect output", $dqstats_id, $PS_EXIT_SYS_ERROR ); +# $ipprc->redirect_output($logDest) +# or my_die( "Unable to redirect output", $dqstats_id, $PS_EXIT_SYS_ERROR ); print "\n\n"; print "Starting script $0 on $host\n\n"; @@ -84,16 +85,16 @@ unless ($no_op) { # This bit needs to make the bundle. - + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $bundle_command, verbose => $verbose); + run(command => $bundle_command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to create the bundle.: $error_code", $dqstats_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to create the bundle.: $error_code", $dqstats_id, $error_code); } - + # Parse stdout_buf to get output file name ## Begin partial my ($tempFile, $tempName) = tempfile( "/tmp/dqstats.XXXX", - UNLINK => !$save_temps, SUFFIX => 'dsin' ); + UNLINK => !$save_temps, SUFFIX => 'dsin' ); print $tempFile $uri . '|||notset' . "\n"; @@ -102,10 +103,10 @@ # It can't be that simple, can it? This should probably add a fault on failure. ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $register_cmd, verbose => $verbose); + run(command => $register_cmd, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to register the bundle.: $error_code", $dqstats_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to register the bundle.: $error_code", $dqstats_id, $error_code); } - + # We no longer need the file generated by dqstatstool. unlink($uri); @@ -118,9 +119,9 @@ # This bit needs to set the database state to full my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $update_command, verbose => $verbose); + run(command => $update_command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to add result to database: $error_code\n"); - exit($error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); } } else { @@ -139,5 +140,5 @@ carp($msg); if (defined $dqstats_id and not $no_update) { - # There is no fault handling, which may be a mistake. This is where a fault coding would be. + # There is no fault handling, which may be a mistake. This is where a fault coding would be. # my $command = "$addtool -add_id $add_id"; # $command .= " -addprocessedexp"; Index: branches/pap/ippScripts/scripts/fake_imfile.pl =================================================================== --- branches/pap/ippScripts/scripts/fake_imfile.pl (revision 27708) +++ branches/pap/ippScripts/scripts/fake_imfile.pl (revision 28003) @@ -24,6 +24,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: branches/pap/ippScripts/scripts/flatcorr_init.pl =================================================================== --- branches/pap/ippScripts/scripts/flatcorr_init.pl (revision 27708) +++ branches/pap/ippScripts/scripts/flatcorr_init.pl (revision 28003) @@ -1,5 +1,5 @@ #!/usr/bin/env perl -## USAGE:flatcorr_init.pl +## USAGE:flatcorr_init.pl ## given dbname, dvodb, filter, time range, camera, telescope, etc? ## select matching images and register as a new flatcorr run @@ -24,6 +24,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -68,7 +69,7 @@ my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); -if ($missing_tools) { +if ($missing_tools) { warn ("Can't find required tools"); - exit($PS_EXIT_CONFIG_ERROR); + exit($PS_EXIT_CONFIG_ERROR); } @@ -90,5 +91,5 @@ cache_run(command => $command, verbose => 1); - unless ($success) { + unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RESORT", $status, $dbname); @@ -99,15 +100,15 @@ { foreach my $filter (@filters) { - my $command = "$relphot $filter"; - $command .= "-D CATDIR $catdir"; - $command .= "-region $RAs $RAe $DECs $DECe"; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => $command, verbose => 1); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); - } + my $command = "$relphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } } } @@ -117,15 +118,15 @@ if (0) { foreach my $filter (@filters) { - my $command = "$uniphot $filter"; - $command .= "-D CATDIR $catdir"; - $command .= "-region $RAs $RAe $DECs $DECe"; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => $command, verbose => 1); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "UNIPHOT", $status, $dbname); - } + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "UNIPHOT", $status, $dbname); + } } } @@ -137,9 +138,9 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => $command, verbose => 1); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); } } @@ -151,9 +152,9 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => $command, verbose => 1); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); } } @@ -185,14 +186,14 @@ my $region = shift; my $last_step = shift; - my $status = shift; - my $dbname = shift; + my $status = shift; + my $dbname = shift; carp($msg); if (defined $dvo_id && defined $region && defined $last_step && defined $status and not $no_update) { my $command = "$caltool -addcalrun"; - $command .= " -dvo_id $dvo_id"; + $command .= " -dvo_id $dvo_id"; $command .= " -region $region"; - $command .= " -last_step $last_step"; - $command .= " -status $status"; + $command .= " -last_step $last_step"; + $command .= " -status $status"; $command .= " -dbname $dbname" if defined $dbname; system ($command); Index: branches/pap/ippScripts/scripts/flatcorr_proc.pl =================================================================== --- branches/pap/ippScripts/scripts/flatcorr_proc.pl (revision 27708) +++ branches/pap/ippScripts/scripts/flatcorr_proc.pl (revision 28003) @@ -11,7 +11,7 @@ # dvoMakeCorr -file outgrid.fits -ref ref.fits outcorr -# dettool -register -det_type FLATCORR -filelevel (level) -workdir -inst, etc - -# foreach $imfile () +# dettool -register -det_type FLATCORR -filelevel (level) -workdir -inst, etc + +# foreach $imfile () # dettool -register_imfile -uri, etc, etc @@ -23,6 +23,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -74,11 +75,11 @@ my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); my $dvoMakeCorr = can_run('dvoMakeCorr') or (warn "Can't find dvoMakeCorr" and $missing_tools = 1); -my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); -my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); -my $flatcorr = can_run('flatcorr') or (warn "Can't find flatcorr" and $missing_tools = 1); - -if ($missing_tools) { +my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $flatcorr = can_run('flatcorr') or (warn "Can't find flatcorr" and $missing_tools = 1); + +if ($missing_tools) { warn ("Can't find required tools"); - exit($PS_EXIT_CONFIG_ERROR); + exit($PS_EXIT_CONFIG_ERROR); } @@ -95,5 +96,5 @@ $ipprc->outroot_prepare($outcorr); -if (not -e "$dvodb/flatcorr") { +if (not -e "$dvodb/flatcorr") { mkdir "$dvodb/flatcorr" or &my_die ("Unable to make output directory for relphot $dvodb/flatcorr", $corr_id, 3); } @@ -104,17 +105,17 @@ my ($DECs, $DECe) = split (",", $coords[1]); -# Run addstar -resort to ensure the db is indexed +# Run addstar -resort to ensure the db is indexed # XXX addstar should be able to recognize and skip indexed tables { - my $command = "$addstar -resort"; + my $command = "$addstar -resort"; $command .= " -D CATDIR $dvodb"; $command .= " -region $RAs $RAe $DECs $DECe"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform addstar -resort for dvodb $dvodb on region $region: $error_code", $corr_id, $error_code); + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort for dvodb $dvodb on region $region: $error_code", $corr_id, $error_code); } } @@ -135,9 +136,9 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform relphot -grid for dvodb $dvodb on region $region: $error_code", $corr_id, $error_code); + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform relphot -grid for dvodb $dvodb on region $region: $error_code", $corr_id, $error_code); } } @@ -176,5 +177,5 @@ # Push the results into the database -{ +{ my $command = "$flatcorr -addprocess"; $command .= " -corr_id $corr_id"; @@ -184,13 +185,13 @@ unless ($no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn ("Unable to perform regtool -addprocessedimfile: $error_code"); - exit($error_code); - } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } } else { - print "skipping command: $command\n"; + print "skipping command: $command\n"; } } @@ -218,5 +219,5 @@ &my_die("Unable to parse metadata list", $corr_id, $PS_EXIT_PROG_ERROR); - # check for existence + # check for existence my $file = $$files[0]; my $chip_id = $file->{chip_id}; @@ -228,5 +229,5 @@ # flatcorr -inputimfile -chip_id $chip_id sub get_imfiles { - my $chip_id = shift; + my $chip_id = shift; my $command = "$flatcorr -inputimfile"; @@ -273,9 +274,9 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to register new detrend: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to register new detrend: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); } @@ -298,39 +299,39 @@ foreach my $file (@$files) { - # create the detrend correction for the imfiles based on the input imfiles - my $reffile = $file->{uri}; - my $class_id = $file->{class_id}; - - my $uri = $ipprc->filename("DVOCORR.OUTPUT", $outcorr, $class_id); - unless ($uri) { - &my_die ("Unable to find DVOCORR.OUTPUT in filerules", $corr_id, $PS_EXIT_PROG_ERROR); - } - - my $command = "$dvoMakeCorr $outcorr"; - $command .= " -file $outgrid.fits"; - $command .= " -ref $reffile"; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to perform dvoMakeCorr: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); - } - - # register the detrend correction imfile - $command = "$dettool -register_detrend_imfile"; - $command .= " -det_id $det_id"; - $command .= " -class_id $class_id"; - $command .= " -uri $uri"; - $command .= " -dbname $dbname" if defined $dbname; - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die ("Unable to register new detrend: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); - } + # create the detrend correction for the imfiles based on the input imfiles + my $reffile = $file->{uri}; + my $class_id = $file->{class_id}; + + my $uri = $ipprc->filename("DVOCORR.OUTPUT", $outcorr, $class_id); + unless ($uri) { + &my_die ("Unable to find DVOCORR.OUTPUT in filerules", $corr_id, $PS_EXIT_PROG_ERROR); + } + + my $command = "$dvoMakeCorr $outcorr"; + $command .= " -file $outgrid.fits"; + $command .= " -ref $reffile"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform dvoMakeCorr: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); + } + + # register the detrend correction imfile + $command = "$dettool -register_detrend_imfile"; + $command .= " -det_id $det_id"; + $command .= " -class_id $class_id"; + $command .= " -uri $uri"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to register new detrend: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); + } } } @@ -345,5 +346,5 @@ if (not $no_update) { my $command = "$flatcorr -addprocess"; - $command .= " -corr_id $corr_id"; + $command .= " -corr_id $corr_id"; $command .= " -fault $exit_code"; $command .= " -hostname $host" if defined $host; Index: branches/pap/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/pap/ippScripts/scripts/ipp_cleanup.pl (revision 27708) +++ branches/pap/ippScripts/scripts/ipp_cleanup.pl (revision 28003) @@ -14,5 +14,4 @@ use File::Spec; use PS::IPP::Config 1.01 qw( :standard ); -use PS::IPP::Metadata::List qw( parse_md_list ); use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); use Pod::Usage qw( pod2usage ); @@ -108,10 +107,7 @@ } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "chip", $stage_id, $PS_EXIT_PROG_ERROR); # loop over all of the imfiles, determine the path_base and class_id for each @@ -196,5 +192,5 @@ # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -249,9 +245,6 @@ &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "camera", $stage_id, $PS_EXIT_PROG_ERROR); - - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "camera", $stage_id, $PS_EXIT_PROG_ERROR); my $n_exps = @$exps; @@ -316,5 +309,5 @@ # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -375,9 +368,6 @@ exit(0); } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $skyfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR); - - $skyfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "warp", $stage_id, $PS_EXIT_PROG_ERROR); my @files = (); @@ -457,5 +447,5 @@ # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -515,9 +505,6 @@ } - my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + $skyfiles = $mdcParser->parse_list(join "", @{ $stdout_buf }) or &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR); - - $skyfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "stack", $stage_id, $PS_EXIT_PROG_ERROR); my @files = (); @@ -594,5 +581,5 @@ # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -652,9 +639,6 @@ } - my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + $skyfiles = $mdcParser->parse_list(join "", @{ $stdout_buf }) or &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); - - $skyfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "diff", $stage_id, $PS_EXIT_PROG_ERROR); my @files = (); @@ -747,5 +731,5 @@ # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -806,10 +790,7 @@ } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR); # loop over all of the imfiles, determine the path_base and class_id for each @@ -927,10 +908,7 @@ - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); } # loop over all of the imfiles, determine the path_base and class_id for each @@ -1013,8 +991,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1068,10 +1044,7 @@ # if there are no detResidImfiles (@$stdout_buf == 0), then silently exit. if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); } # loop over all of the imfiles, determine the path_base and class_id for each @@ -1156,8 +1129,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1202,9 +1173,6 @@ &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); } - my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + $stacks = $mdcParser->parse_list(join "", @{ $stdout_buf }) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - $stacks = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); my @files = (); @@ -1286,8 +1254,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1327,9 +1293,6 @@ &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1380,8 +1343,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1425,9 +1386,6 @@ &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1511,8 +1469,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1559,9 +1515,6 @@ exit 0; } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { @@ -1643,8 +1596,6 @@ } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $exps = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); foreach my $exp (@$exps) { Index: branches/pap/ippScripts/scripts/ipp_detrend_combine.pl =================================================================== --- branches/pap/ippScripts/scripts/ipp_detrend_combine.pl (revision 27708) +++ branches/pap/ippScripts/scripts/ipp_detrend_combine.pl (revision 28003) @@ -7,6 +7,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -32,24 +33,24 @@ $workdir, $dbname, $no_update); GetOptions( - 'det_type=s' => \$det_type, # Detrend type for new detrend - 'filelevel=s' => \$filelevel, # File level for new detrend - 'inst=s' => \$inst, # Instrument for new detrend - 'telescope=s' => \$telescope, # Telescope for new detrend - 'filter=s' => \$filter, # Filter name for new detrend - 'det_id1=s' => \$det_id1, # Detrend id for detrend 1 - 'iteration1=s' => \$iter1, # Iteration for detrend 1 - 'det_id2=s' => \$det_id2, # Detrend id for detrend 2 - 'iteration2=s' => \$iter2, # Iteration for detrend 2 - 'operation=s' => \$operation, # Operation to perform on files - 'mask' => \$mask, # Operation is on a mask - 'workdir=s' => \$workdir, # Working directory for output files - 'dbname=s' => \$dbname, # Database name - 'no-update' => \$no_update, # Don't update the database - ) or pod2usage( 2 ); + 'det_type=s' => \$det_type, # Detrend type for new detrend + 'filelevel=s' => \$filelevel, # File level for new detrend + 'inst=s' => \$inst, # Instrument for new detrend + 'telescope=s' => \$telescope, # Telescope for new detrend + 'filter=s' => \$filter, # Filter name for new detrend + 'det_id1=s' => \$det_id1, # Detrend id for detrend 1 + 'iteration1=s' => \$iter1, # Iteration for detrend 1 + 'det_id2=s' => \$det_id2, # Detrend id for detrend 2 + 'iteration2=s' => \$iter2, # Iteration for detrend 2 + 'operation=s' => \$operation, # Operation to perform on files + 'mask' => \$mask, # Operation is on a mask + 'workdir=s' => \$workdir, # Working directory for output files + 'dbname=s' => \$dbname, # Database name + 'no-update' => \$no_update, # Don't update the database + ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( -msg => "Required options --det_type --filelevel --inst --telescope --det_id1 --iteration1 --det_id2 --iteration2 --workdir", - -exitval => 3, - ) + -exitval => 3, + ) unless defined $det_type and defined $filelevel @@ -65,6 +66,6 @@ $ipprc->define_camera($inst); -my $STATS = - [ +my $STATS = + [ # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, @@ -90,15 +91,15 @@ die("File lists for detrends have differing lengths") unless scalar keys %$files1 == scalar keys %$files2; -my ($det_id, $iter); # Detrend identifier for the new detrend +my ($det_id, $iter); # Detrend identifier for the new detrend unless ($no_update) { my $command = "$dettool -register_detrend -det_type $det_type -filelevel $filelevel -workdir $workdir " . - "-inst $inst -telescope $telescope"; # Command to run + "-inst $inst -telescope $telescope"; # Command to run $command .= " -filter $filter" if defined $filter; $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); + run(command => $command, verbose => 1); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - die("Unable to run dettool -register_detrend: $error_code"); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend: $error_code"); } @@ -137,8 +138,8 @@ $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); + run(command => $command, verbose => 1); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - die("Unable to run ppArith: $error_code"); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run ppArith: $error_code"); } @@ -149,28 +150,28 @@ my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser { - my $statsFile; # File handle - open $statsFile, $ipprc->file_resolve($outStats) or die("Can't open stats file $outStats: $!"); - my @contents = <$statsFile>; # Contents of file - close $statsFile; - - my $metadata = $mdcParser->parse(join "", @contents) or die("Unable to parse metadata config doc"); - - unless ($stats->parse($metadata)) { - &my_die("Failure extracting metadata from the statistics output file.\n"); - } + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outStats) or die("Can't open stats file $outStats: $!"); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + my $metadata = $mdcParser->parse(join "", @contents) or die("Unable to parse metadata config doc"); + + unless ($stats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n"); + } } # Register the imfile unless ($no_update) { - my $command = "$dettool -register_detrend_imfile -det_id $det_id "; # Command to run - $command .= " -class_id $class_id -uri $outName -path_base $outRoot"; - $command .= $stats->cmdflags(); - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - die("Unable to run dettool -register_detrend_imfile: $error_code"); - } + my $command = "$dettool -register_detrend_imfile -det_id $det_id "; # Command to run + $command .= " -class_id $class_id -uri $outName -path_base $outRoot"; + $command .= $stats->cmdflags(); + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend_imfile: $error_code"); + } } } @@ -183,14 +184,14 @@ sub filelist { - my $det_id = shift; # Detrend identifier - my $iter = shift; # Iteration + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration my $command = "$detselect -select -det_id $det_id -iteration $iter"; # Command to run $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); + run(command => $command, verbose => 1); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - die("Unable to run detselect: $error_code"); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run detselect: $error_code"); } @@ -202,8 +203,8 @@ foreach my $item ( @$list ) { - my $class_id = $item->{class_id}; - die("Multiple definitions of class_id=$class_id found for det_id=$det_id, iteration=$iter\n") if - defined $files{$class_id}; - $files{$class_id} = parse_md_list($md); + my $class_id = $item->{class_id}; + die("Multiple definitions of class_id=$class_id found for det_id=$det_id, iteration=$iter\n") if + defined $files{$class_id}; + $files{$class_id} = parse_md_list($md); } Index: branches/pap/ippScripts/scripts/lossy_compress_imfile.pl =================================================================== --- branches/pap/ippScripts/scripts/lossy_compress_imfile.pl (revision 27708) +++ branches/pap/ippScripts/scripts/lossy_compress_imfile.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -86,36 +87,36 @@ if ($state eq 'goto_compressed') { # Find the actual filename for this run: - &my_die("Couldn't find input file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) - unless ($ipprc->file_exists($uri)); + &my_die("Couldn't find input file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + unless ($ipprc->file_exists($uri)); my $uriReal = $ipprc->file_resolve( $uri ); # Create a compressed version: my $compUri = $uri . ".fz"; - &my_die("Output compressed file already exists: $compUri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) - if ($ipprc->file_exists($compUri)); + &my_die("Output compressed file already exists: $compUri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + if ($ipprc->file_exists($compUri)); my $compReal= $ipprc->file_resolve( $compUri, 'create'); # Apparently we need to funpack before fpacking. Probably should have realized that beforehand. my $tempfile = new File::Temp ( TEMPLATE => "${exp_name}.XXXX", - DIR => '/tmp/', - UNLINK => !$save_temps, - SUFFIX => '.fits'); + DIR => '/tmp/', + UNLINK => !$save_temps, + SUFFIX => '.fits'); my $tempReal = $tempfile->filename; my $uncompress_command = "$funpack -S $uriReal > $tempReal"; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $uncompress_command, verbose => $verbose); + run(command => $uncompress_command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to uncompress file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to uncompress file: $uri -> $compUri: $error_code", + $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); } my $compress_command = "$fpack -h -s 8 -S $tempReal > $compReal"; print STDERR "$compReal $uriReal $compress_command\n"; ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $compress_command, verbose => $verbose); + run(command => $compress_command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to compress file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to compress file: $uri -> $compUri: $error_code", + $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); } my $database_command = "$regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state compressed"; @@ -123,17 +124,17 @@ &my_die("Expected output compressed file not found: $compUri\n", $exp_id,$exp_name,$class_id, $uri,$PS_EXIT_SYS_ERROR) - unless ($ipprc->file_exists($compUri)); + unless ($ipprc->file_exists($compUri)); if ($no_update || $no_op) { - print STDERR "NOUPDATE: $database_command\n"; + print STDERR "NOUPDATE: $database_command\n"; } else { - ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $database_command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to update database file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); - } + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $database_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to update database file: $uri -> $compUri: $error_code", + $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + } } exit(0); @@ -149,28 +150,28 @@ # Confirm we have both files &my_die("Couldn't find original file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) - unless($ipprc->file_exists($uri)); + unless($ipprc->file_exists($uri)); my $uriReal = $ipprc->file_resolve( $uri ); - + my $compUri = $uri . ".fz"; unless($ipprc->file_exists($compUri)) { - &my_die("Couldn't find compressed version of the file: $compUri\n", - $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); - - # If that die is removed, this will compress things as well. - # -# &my_die("Output compressed file already exists: $compUri\n", -# $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) -# if ($ipprc->file_exists($compUri)); -# my $compReal= $ipprc->file_resolve( $compUri, 'create'); - -# my $compress_command = "$fpack -h -s 8 -S $uriReal > $compReal"; -# print STDERR "$compReal $uriReal $compress_command\n"; -# my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = -# run(command => $compress_command, verbose => $verbose); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to compress file: $uri -> $compUri: $error_code", -# $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); -# } + &my_die("Couldn't find compressed version of the file: $compUri\n", + $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + + # If that die is removed, this will compress things as well. + # +# &my_die("Output compressed file already exists: $compUri\n", +# $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) +# if ($ipprc->file_exists($compUri)); +# my $compReal= $ipprc->file_resolve( $compUri, 'create'); + +# my $compress_command = "$fpack -h -s 8 -S $uriReal > $compReal"; +# print STDERR "$compReal $uriReal $compress_command\n"; +# my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = +# run(command => $compress_command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to compress file: $uri -> $compUri: $error_code", +# $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); +# } } @@ -181,10 +182,10 @@ unless ($no_op) { - $neb->replicate($compUri); - $neb->swap($uri,$compUri) or - &my_die("Nebulous swap failed between $uri and $compUri", - $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); - } - # Update database + $neb->replicate($compUri); + $neb->swap($uri,$compUri) or + &my_die("Nebulous swap failed between $uri and $compUri", + $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + } + # Update database my $database_command = "$regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state lossy "; @@ -192,20 +193,20 @@ $database_command .= " -dbname $dbname" if defined $dbname; - + if ($no_update || $no_op) { - print STDERR "NOUPDATE: $database_command\n"; + print STDERR "NOUPDATE: $database_command\n"; } else { - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $database_command, verbose => $verbose); - unless($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - $neb->swap($uri,$compUri) or - &my_die("DB update failed and Nebulous swap-back failed between $uri and $compUri", - $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); - &my_die("Unable to update database file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); - } - # remove original version + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $database_command, verbose => $verbose); + unless($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + $neb->swap($uri,$compUri) or + &my_die("DB update failed and Nebulous swap-back failed between $uri and $compUri", + $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + &my_die("Unable to update database file: $uri -> $compUri: $error_code", + $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + } + # remove original version $neb->delete($compUri); } Index: branches/pap/ippScripts/scripts/magic_destreak.pl =================================================================== --- branches/pap/ippScripts/scripts/magic_destreak.pl (revision 27708) +++ branches/pap/ippScripts/scripts/magic_destreak.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -41,5 +42,5 @@ # Parse the command-line arguments my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction); -my ($outroot, $recoveryroot); +my ($outroot, $recoveryroot, $magicked); my ($replace, $release); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); @@ -61,4 +62,5 @@ 'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels 'replace=s' => \$replace, # replace the input images with the results. + 'magicked=s' => \$magicked, # magicked state of the run 'release' => \$release, # NAN masked pixels for release 'save-temps' => \$save_temps, # Save temporary files? @@ -71,5 +73,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --uri --path_base --outroot", +pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --uri --path_base --outroot --magicked", -exitval => 3) unless defined $magic_ds_id and @@ -82,5 +84,6 @@ defined $uri and defined $path_base and - defined $outroot; + defined $outroot and + defined $magicked; my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -209,5 +212,4 @@ &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); - $temp_dir = tempdir( CLEANUP => !$save_temps); ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps); @@ -218,4 +220,7 @@ } else { # diff run must have been cleaned up, need to create this skycell file on the fly + if (!defined $temp_dir ) { + $temp_dir = tempdir( CLEANUP => !$save_temps); + } my $skycell_id = $skycell->{skycell_id}; $skycell_uri = "$temp_dir/$skycell_id"; @@ -247,5 +252,6 @@ my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + # note verbose == 0 to avoid polluting log files with almost always useless information + run(command => $command, verbose => 0); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -266,5 +272,7 @@ $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base); - if ($dynamicMasks) { + # if magicked is non-zero we are updating a previously magicked component. In this case we don't + # touch the camera mask + if (!$magicked and $dynamicMasks) { $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id); $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id); Index: branches/pap/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- branches/pap/ippScripts/scripts/magic_destreak_cleanup.pl (revision 27708) +++ branches/pap/ippScripts/scripts/magic_destreak_cleanup.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -62,13 +63,4 @@ $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile; -if (0) { -my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' ); -&my_die("cannot find NEB_SERVER in site configuration", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer; - -my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); }; -if ($@ or not defined $nebulous) { - &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); -} -} $dbname = metadataLookupStr( $ipprc->{_siteConfig}, 'DBNAME' ) if !$dbname; @@ -107,4 +99,7 @@ +&my_die("cleanup not supported for camera stage", $magic_ds_id, $PS_EXIT_PROG_ERROR) if $stage eq "camera"; + + &my_die("unexpected run state found: $state", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $state ne "goto_cleaned"; &my_die("clean not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw"; @@ -129,4 +124,8 @@ $dbh->disconnect() or warn $dbh->errstr; +# We no longer clean up the camera stage mask files, but the code was left in place in case +# we change our minds. +my $cleanup_cam_mask = 0; + my $dynamicMasks; # Use dynamic masks? foreach my $comp (@components) { @@ -139,5 +138,5 @@ if ($stage eq "chip") { # Check to see if we're using dynamic masks - if (!defined $dynamicMasks) { + if ($cleanup_cam_mask && !defined $dynamicMasks) { # Get the PSASTRO recipe my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; Index: branches/pap/ippScripts/scripts/magic_destreak_defineruns.pl =================================================================== --- branches/pap/ippScripts/scripts/magic_destreak_defineruns.pl (revision 27708) +++ branches/pap/ippScripts/scripts/magic_destreak_defineruns.pl (revision 28003) @@ -11,6 +11,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- branches/pap/ippScripts/scripts/magic_destreak_revert.pl (revision 27708) +++ branches/pap/ippScripts/scripts/magic_destreak_revert.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -37,5 +38,5 @@ # Parse the command-line arguments -my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction); +my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction, $magicked); my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); @@ -54,4 +55,5 @@ 'replace=s' => \$replace, # replace the input images with the results. 'bothways=s' => \$bothways, # run has inverse files (bothways diff) + 'magicked=s' => \$magicked, # magicked state of the run 'save-temps' => \$save_temps, # Save temporary files? 'dbname=s' => \$dbname, # Database name @@ -63,5 +65,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --magic_ds_id --camera --stage --stage_id --component --path_base --outroot", +pod2usage( -msg => "Required options: --magic_ds_id --camera --stage --stage_id --component --path_base --outroot --magicked", -exitval => 3) unless defined $magic_ds_id and @@ -71,4 +73,5 @@ defined $component and defined $path_base and + defined $magicked and defined $outroot; @@ -76,4 +79,6 @@ &my_die("cam_path_base is required for chip stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'chip' and !$cam_path_base); + + my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -168,5 +173,6 @@ my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + # note verbose == 0 to keep from polluting log file with lots of usually useless information + run(command => $command, verbose => 0); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -181,10 +187,12 @@ } + $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id); + $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id); + # we use the mask output from the camera stage for input and replace # the output of the chip stage with that mask as well. - $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id); - $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id); - - if ($dynamicMasks) { + # Note that when destreaking as part of an update $magicked is non-zero. + # In this case we do not touch the camera stage mask so there is no need to revert it + if (!$magicked and $dynamicMasks) { $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id); $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id); @@ -194,5 +202,5 @@ $bimage = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $class_id); - # This is kludgey but correct + # This is somewhat kludgey but it works whether the mask is camera mask or chip mask $bmask = dirname($backup_path_base) . "/SR_" . basename($mask); $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id); Index: branches/pap/ippScripts/scripts/magic_process.pl =================================================================== --- branches/pap/ippScripts/scripts/magic_process.pl (revision 27708) +++ branches/pap/ippScripts/scripts/magic_process.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -98,13 +99,11 @@ } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + $inputs = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $magic_id, $node, $PS_EXIT_PROG_ERROR); - - $inputs = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $magic_id, $node, $PS_EXIT_PROG_ERROR); } my @outputs; +my $inverse; # Using inverse diff? ### Do the heavy lifting { @@ -262,4 +261,8 @@ cat_list_to_list($mfh, $in_path_base, "mask.list"); cat_list_to_list($wfh, $in_path_base, "weight.list"); + + if ($innode->{inverse}) { + $inverse = 1; + } } close $in_fh; @@ -329,4 +332,5 @@ } } + if ($node eq "root") { my $streaks_file = "$outroot.streaks"; @@ -347,21 +351,56 @@ &run_verifystreaks($baseroot); - my $command = "$magictool -addmask"; - $command .= " -magic_id $magic_id"; - $command .= " -uri $streaks_file"; - $command .= " -streaks $num_streaks"; - $command .= " -dbname $dbname" if defined $dbname; - - # Add the processed file to the database - unless ($no_update) { + my $exp_id; # Exposure identifier + my $cam_path; # Camera stage path_base + { + my $command = "magictool -exposure -magic_id $magic_id"; + $command .= " -inverse" if defined $inverse; + $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - # This isn't going to work because our result was already entered. - &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code); - } - } else { - print "Skipping command: $command\n"; + my $exposures = $mdcParser->parse_list(join "", @$stdout_buf); + my $exp = $$exposures[0]; # Exposure of interest (should only be one) + + $exp_id = $exp->{exp_id}; + $cam_path = $exp->{path_base}; + } + + { + my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file + my $streaks = "$outroot.streaks"; # Streaks file + my $clusters = "$baseroot.verify/${exp_id}_clusterPos.txt"; # Clusters file + + my $command = "ppCoord -astrom $astrom -streaks $streaks"; + if ($ipprc->file_exists($clusters)) { + $command .= " -clusters $clusters"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + open my $coords, "> $outroot.coords"; + print $coords join("", @$stdout_buf); + close $coords; + } + + + { + my $command = "$magictool -addmask"; + $command .= " -magic_id $magic_id"; + $command .= " -uri $streaks_file"; + $command .= " -streaks $num_streaks"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + # This isn't going to work because our result was already entered. + &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code); + } + } else { + print "Skipping command: $command\n"; + } } } Index: branches/pap/ippScripts/scripts/magic_tree.pl =================================================================== --- branches/pap/ippScripts/scripts/magic_tree.pl (revision 27708) +++ branches/pap/ippScripts/scripts/magic_tree.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/make_burntool_pcontrol.pl =================================================================== --- branches/pap/ippScripts/scripts/make_burntool_pcontrol.pl (revision 27708) +++ branches/pap/ippScripts/scripts/make_burntool_pcontrol.pl (revision 28003) @@ -1,8 +1,12 @@ -#!/usr/bin/perl -w +#!/usr/bin/env perl # script to generate a pcontrol.pro file to run burntool on a set of data. use DBI; +use warnings; use DateTime; use Getopt::Std; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use IPC::Cmd 0.36 qw( can_run run); use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; @@ -35,7 +39,11 @@ ) or die "Unable to connect to database $DBI::errstr\n"; +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + die "Cannot find required tools."; +} # Determine what the value of "BURNTOOL.STATE.GOOD" currently is: $burntoolStateGood = 999; -open(LAZY,"ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n"; +open(LAZY,"$ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n"; while() { chomp; @@ -46,14 +54,54 @@ } close(LAZY); -#unless( $burntoolStateGood == 999) { -# print STDERR "GOOD == $burntoolStateGood\n"; -#} - - -# List of which obs_modes are "science" and which are something else. -%science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1, - 'ENGINEERING' => 0, 'NULL' => 0, 'Unknown' => 1, ' ' => 0); + +# Read the nightly science config file and use that to determine which data is science +my $verbose = 0; +my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR); +} + +my @target_list; +my %object_list; +my $comment_list; + +my $mdcParser = PS::IPP::Metadata::Config->new; +my $metadata = $mdcParser->parse(join "", @$stdout_buf); +foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'TARGETS') { + my @target_data = @{ ${ $entry }{value} }; + my $this_target = ''; + + foreach my $tentry (@target_data) { + if (${ $tentry }{name} eq 'NAME') { + $this_target = ${ $tentry }{value}; + push @target_list, $this_target; + $obsmode_list{$this_target} = '.*'; + $object_list{$this_target} = '.*'; + $comment_list{$this_target} = '.*'; + } + elsif (${ $tentry }{name} eq 'OBSMODE') { + $obsmode_list{$this_target} = ${ $tentry }{value}; + $obsmode_list{$this_target} =~ s/%//; + } + elsif (${ $tentry }{name} eq 'OBJECT') { + $object_list{$this_target} = ${ $tentry }{value}; + $object_list{$this_target} =~ s/%//; + } + elsif (${ $tentry }{name} eq 'COMMENT') { + $comment_list{$this_target} = ${ $tentry }{value}; + $comment_list{$this_target} =~ s/%//; + } + } + } +} if (exists($opt{P})) { - $science{Unknown} = 1; + push @target_list, 'PR'; + $obsmode_list{'PR'} = 'Unknown'; + $object_list{'PR'} = '.*'; + $comment_list{'PR'} = '.*'; } @@ -119,5 +167,5 @@ # Find _ALL_ observations within the date range we care about - $sth = "SELECT exp_id,dateobs,pon_time,exp_type,obs_mode,comment FROM rawExp WHERE " . + $sth = "SELECT exp_id,dateobs,pon_time,exp_type,obs_mode,comment,object FROM rawExp WHERE " . "dateobs >= '${date_min}' AND dateobs <= '${date_max}' order by dateobs"; @@ -133,9 +181,12 @@ foreach $row_ref (@{ $data_ref }) { - ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment) = @{ $row_ref }; + ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment,$object) = @{ $row_ref }; # Fix nulls in the database if (!defined($obs_mode)) { $obs_mode = ' '; } + if (!defined($object)) { + $object = ' '; + } if (!defined($comment)) { $comment = ' '; @@ -144,5 +195,5 @@ ($date,$time) = split / /, $dateobs; ($year,$month,$day) = split /-/, $date; - ($hour,$minute,$second) = split /:/, $time; + ($hour,$minute,$second) = split /:/, $time; # / # This stops emacs run-on syntax highlights. $dt = DateTime->new( year => $year, month => $month, day => $day, @@ -155,16 +206,5 @@ $et = $dt->epoch(); - - if (!exists($science{$obs_mode})) { - $science{$obs_mode} = 0; - } - if ((($science{$obs_mode} == 1)&&($comment !~ /Daytime/))|| - (($science{$obs_mode} == 0)&&( - ($comment =~ /MD/)||($comment =~ /ThreePi/)|| - ($comment =~ /STS/i)||($comment =~ /M31/)||($comment =~ /Stellar Transit/)|| - ($comment =~ /CAL/i)||($comment =~ /SS/)||($comment =~ /SVS/)|| - ($comment =~ /Sweetspot/)||($comment =~ /virgo/i)||($comment =~ /kepler/)|| - ($comment =~ /sdss/)))) - { + if (is_science($exp_type,$obs_mode,$object,$comment)) { if ($start_s[-1] == 0) { $start_s[-1] = $et - 1800; @@ -194,5 +234,5 @@ # Scan again and count how many exposures are between windows foreach $row_ref (@{ $data_ref }) { - ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment) = @{ $row_ref }; + ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment,$object) = @{ $row_ref }; if (!defined($obs_mode)) { $obs_mode = ' '; @@ -204,5 +244,5 @@ ($date,$time) = split / /, $dateobs; ($year,$month,$day) = split /-/, $date; - ($hour,$minute,$second) = split /:/, $time; + ($hour,$minute,$second) = split /:/, $time; # / # another emacs syntax highlight bug. $dt = DateTime->new( year => $year, month => $month, day => $day, @@ -215,10 +255,6 @@ $et = $dt->epoch(); - if (!exists($science{$obs_mode})) { - die "No mode >>$obs_mode<<\n"; - } - for ($i = 0; $i <= $#start_s; $i++) { - if (($et < $start_s[$i])&&($science{$obs_mode} == 0)) { + if (($et < $start_s[$i])&&(is_science($exp_type,$obs_mode,$object,$comment) == 0)) { if ($i == 0) { $skips[$i]++; @@ -278,4 +314,30 @@ unless(exists($opt{b}) || ($N_ranges == 0)) { print_epilogue(); +} + + +# Subroutine to use the configuration file data and the exposure data to determine whether or not something is "science" +sub is_science { +# if (is_science($exp_type,$obs_mode,$object,$comment)) { + my ($exp_type,$obs_mode,$object,$comment) = @_; + my $return_value = 0; + unless($object) { + $object = ''; + } + if ($exp_type eq 'OBJECT') { + foreach my $target (@target_list) { +# print STDERR "$target $obsmode_list{$target} $exp_type $obs_mode $object $comment\n"; + if (($obs_mode =~ /$obsmode_list{$target}/)&& + ($object =~ /$object_list{$target}/)&& + ($comment =~ /$comment_list{$target}/)) { + $return_value = 1; + } +# print ">$return_value $exp_type $obs_mode $object $comment $target $obsmode_list{$target} $object_list{$target} $comment_list{$target}\n"; + if ($return_value) { + return($return_value); + } + } + } + return(0); } Index: branches/pap/ippScripts/scripts/publish_file.pl =================================================================== --- branches/pap/ippScripts/scripts/publish_file.pl (revision 27708) +++ branches/pap/ippScripts/scripts/publish_file.pl (revision 28003) @@ -7,5 +7,7 @@ use Sys::Hostname; my $host = hostname(); -print "Starting script $0 on $host\n\n"; +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: branches/pap/ippScripts/scripts/rcserver_checkstatus.pl =================================================================== --- branches/pap/ippScripts/scripts/rcserver_checkstatus.pl (revision 27708) +++ branches/pap/ippScripts/scripts/rcserver_checkstatus.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/receive_advance.pl =================================================================== --- branches/pap/ippScripts/scripts/receive_advance.pl (revision 27708) +++ branches/pap/ippScripts/scripts/receive_advance.pl (revision 28003) @@ -7,5 +7,7 @@ use Sys::Hostname; my $host = hostname(); -print "Starting script $0 on $host\n\n"; +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; @@ -58,7 +60,6 @@ &my_die( "Unable to set up", $fileset_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -# update the fileset entry - -if ($dbinfo_uri and ($dbinfo_uri ne "NULL")) { +my $import_run_to_db = 0; +if ($import_run_to_db and $dbinfo_uri and ($dbinfo_uri ne "NULL")) { my $filename = basename($dbinfo_uri); my ($stage) = $filename =~ m|^dbinfo\.(\S+)\.\d+\.mdc$|; # Stage of interest @@ -100,4 +101,5 @@ } +# update the fileset entry # All done { Index: branches/pap/ippScripts/scripts/receive_file.pl =================================================================== --- branches/pap/ippScripts/scripts/receive_file.pl (revision 27708) +++ branches/pap/ippScripts/scripts/receive_file.pl (revision 28003) @@ -7,5 +7,7 @@ use Sys::Hostname; my $host = hostname(); -print "Starting script $0 on $host\n\n"; +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; @@ -124,5 +126,5 @@ print "dirinfo resolved is: $resolved\n" if $verbose; - open OUT, ">$resolved" + open OUT, ">$resolved" or &my_die( "failed to open $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR); print OUT @$dirinfo_lines @@ -130,5 +132,5 @@ close OUT or &my_die( "failed to close $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR); - + # update the fileset to allow processing of other files my $command = "$receivetool -updatefileset -fileset_id $fileset_id"; @@ -160,5 +162,5 @@ &my_die( "failed to resolve $dbinfo_uri\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$resolved; - open OUT, ">$resolved" + open OUT, ">$resolved" or &my_die( "failed to open $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR); @@ -227,5 +229,5 @@ $current_component = $value; $component_dir = $components->{$current_component}; - &my_die( "$component_dir is null for $value in $filename: $runType\n", + &my_die( "$component_dir is null for $value in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$component_dir; } elsif ($name eq 'workdir') { @@ -237,5 +239,5 @@ $new_value = basename($value); } elsif ((($name eq 'uri') or ($name eq 'path_base')) and ($value ne 'NULL')) { - &my_die( "$component_dir is null and we need it for $name", + &my_die( "$component_dir is null and we need it for $name", $file_id, $PS_EXIT_PROG_ERROR) if !$component_dir; Index: branches/pap/ippScripts/scripts/receive_fileset.pl =================================================================== --- branches/pap/ippScripts/scripts/receive_fileset.pl (revision 27708) +++ branches/pap/ippScripts/scripts/receive_fileset.pl (revision 28003) @@ -7,5 +7,7 @@ use Sys::Hostname; my $host = hostname(); -print "Starting script $0 on $host\n\n"; +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: branches/pap/ippScripts/scripts/receive_setstatus.pl =================================================================== --- branches/pap/ippScripts/scripts/receive_setstatus.pl (revision 27708) +++ branches/pap/ippScripts/scripts/receive_setstatus.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -74,5 +75,5 @@ $fileargs = " --list - --copy --abspath"; } - + # XXX need to create a fileset type for this my $command = "echo $regline | dsreg --add $status_fs_name --product $status_product --type notset"; Index: branches/pap/ippScripts/scripts/receive_source.pl =================================================================== --- branches/pap/ippScripts/scripts/receive_source.pl (revision 27708) +++ branches/pap/ippScripts/scripts/receive_source.pl (revision 28003) @@ -7,5 +7,7 @@ use Sys::Hostname; my $host = hostname(); -print "Starting script $0 on $host\n\n"; +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: branches/pap/ippScripts/scripts/register_exp.pl =================================================================== --- branches/pap/ippScripts/scripts/register_exp.pl (revision 27708) +++ branches/pap/ippScripts/scripts/register_exp.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -126,4 +127,5 @@ $command .= " -end_stage $end_stage" if defined $end_stage; $command .= " -tess_id $tess_id" if defined $tess_id; +$command .= " -state full"; $command .= " $cmdflags"; Index: branches/pap/ippScripts/scripts/register_imfile.pl =================================================================== --- branches/pap/ippScripts/scripts/register_imfile.pl (revision 27708) +++ branches/pap/ippScripts/scripts/register_imfile.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -115,39 +116,39 @@ my $burntoolStateGood = 0; foreach my $line (split /\n/, $out1) { - if ($line =~ /FPA.BURNTOOL.APPLIED/) { - $line =~ s/^\s+//; - $burntoolState = (split /\s+/, $line)[2]; - } - if ($line =~ /FPA.CAMERA/) { - $line =~ s/^\s+//; - if ((split /\s+/, $line)[2] eq 'GPC1') { - $isGPC1 = 1; - } - } + if ($line =~ /FPA.BURNTOOL.APPLIED/) { + $line =~ s/^\s+//; + $burntoolState = (split /\s+/, $line)[2]; + } + if ($line =~ /FPA.CAMERA/) { + $line =~ s/^\s+//; + if ((split /\s+/, $line)[2] eq 'GPC1') { + $isGPC1 = 1; + } + } } if ($isGPC1 != 1) { - $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool. + $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool. } elsif (($isGPC1 == 1) && ($burntoolState == 1)) { -# print STDERR "In the good region: >>$burntoolState<<\n"; - my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - IPC::Cmd::run(command => $ppConfigDump_cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn ("Unable to perform ppConfigDump"); - exit($error_code); - } - - # This is ugly, but doing a full parse for one entry is a bit wasteful. - foreach my $line (split /\n/, (join "", @$stdout_buf)) { - if ($line =~ /BURNTOOL.STATE.GOOD/) { - $line =~ s/^\s+//; - $burntoolStateGood = (split /\s+/, $line)[2]; - last; - } - } - $burntoolState = $burntoolStateGood; # Positive because this has the header table. - +# print STDERR "In the good region: >>$burntoolState<<\n"; + my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + IPC::Cmd::run(command => $ppConfigDump_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform ppConfigDump"); + exit($error_code); + } + + # This is ugly, but doing a full parse for one entry is a bit wasteful. + foreach my $line (split /\n/, (join "", @$stdout_buf)) { + if ($line =~ /BURNTOOL.STATE.GOOD/) { + $line =~ s/^\s+//; + $burntoolStateGood = (split /\s+/, $line)[2]; + last; + } + } + $burntoolState = $burntoolStateGood; # Positive because this has the header table. + } $cmdflags .= " -burntool_state $burntoolState "; @@ -173,4 +174,5 @@ $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; +$command .= " -data_state full"; $command .= " $cmdflags"; Index: branches/pap/ippScripts/scripts/stack_skycell.pl =================================================================== --- branches/pap/ippScripts/scripts/stack_skycell.pl (revision 27708) +++ branches/pap/ippScripts/scripts/stack_skycell.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; Index: branches/pap/ippScripts/scripts/staticsky.pl =================================================================== --- branches/pap/ippScripts/scripts/staticsky.pl (revision 28003) +++ branches/pap/ippScripts/scripts/staticsky.pl (revision 28003) @@ -0,0 +1,324 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); +my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($sky_id, $dbname, $threads, $outroot, $reduction, $inverse, $run_state, $verbose, $no_update, $no_op, $redirect, $save_temps); +GetOptions( + 'sky_id=s' => \$sky_id, # Diff identifier + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'run-state=s' => \$run_state, # state for run: 'new' or 'update' + 'outroot=s' => \$outroot, # Output root name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --sky_id --outroot", + -exitval => 3, + ) unless + defined $sky_id and + defined $outroot; + +my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $sky_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +# XXX camera is not known here; cannot use filerules... +# my $logDest = $ipprc->filename("LOG.EXP", $outroot); +my $logDest = "$outroot.log"; +$logDest .= ".update" if $run_state eq "update"; +$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $sky_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF); + +my $outbase = basename($outroot); +my ($listFile, $listName) = tempfile("/tmp/$outbase.list.XXXX", UNLINK => !$save_temps ); + +# Get list of input images to stack photometry +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$staticskytool -inputs -sky_id $sky_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -inputskyfile: $error_code", $sky_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $sky_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $sky_id, $PS_EXIT_PROG_ERROR); +} + +# XXX do we NEED to define the camera? +# &my_die("Unable to identify camera", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless defined $camera; +# $ipprc->define_camera($camera); + +# generate the input +print $listFile "INPUT MULTI\n"; + +foreach my $file (@$files) { + print $listFile "INPUT METADATA\n"; + + $path_base = $file->{path_base}; + my $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $file->{path_base} ); # Mask name + my $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $file->{path_base} ); # Mask name + my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name + my $psfCnv = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $file->{path_base} ); # PSF name + + my $imageRaw = $ipprc->filename("PPSTACK.UNCONV", $file->{path_base} ); # Mask name + my $maskRaw = $ipprc->filename("PPSTACK.UNCONV.MASK", $file->{path_base} ); # Mask name + my $weightRaw = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $file->{path_base} ); # Weight name + + my $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $file->{path_base}); # Sources name + + &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); + &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask ); + &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $weight ); + &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) if ($convolve and not $ipprc->file_exists( $psf )); + &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $sources ); + + print $listFile " RAW:IMAGE STR " . $imageRaw . "\n"; + print $listFile " RAW:MASK STR " . $maskRaw . "\n"; + print $listFile " RAW:VARIANCE STR " . $weightRaw . "\n"; + + print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; + print $listFile " CNV:MASK STR " . $maskCnv . "\n"; + print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; + print $listFile " CNV:PSF STR " . $psfCnv . "\n"; + + print $listFile " SOURCES STR " . $sources . "\n"; + + print $listFile "END\n\n"; + + &my_die("Couldn't find input: $imageRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageRaw"); + &my_die("Couldn't find input: $maskRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskRaw"); + &my_die("Couldn't find input: $weightRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw"); + &my_die("Couldn't find input: $imageCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv"); + &my_die("Couldn't find input: $maskCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv"); + &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv"); + &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); + &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources"); +} + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT'); # Recipe to use for psphot +unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for STACKPHOT: $reduction\n", $sky_id, $PS_EXIT_CONFIG_ERROR); +} + +print "reduction: $reduction\n"; +print "recipe_psphot: $recipe_psphot\n"; + +# Get the output filenames +my $outputName = $ipprc->filename("PPSUB.OUTPUT", $outroot); +my $outputMask = $ipprc->filename("PPSUB.OUTPUT.MASK", $outroot); +my $outputVariance = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $outroot); +my $outputSources = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $outroot); +my $jpeg1Name = $ipprc->filename("PPSUB.OUTPUT.JPEG1", $outroot); +my $jpeg2Name = $ipprc->filename("PPSUB.OUTPUT.JPEG2", $outroot); +my $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); + +if ($run_state eq 'update') { + $traceDest .= '.update'; + $outputStats .= '.update'; +} + +my ($inverseName, $inverseMask, $inverseVariance, $inverseSources); +if ($inverse) { + $inverseName = $ipprc->filename("PPSUB.INVERSE", $outroot); + $inverseMask = $ipprc->filename("PPSUB.INVERSE.MASK", $outroot); + $inverseVariance = $ipprc->filename("PPSUB.INVERSE.VARIANCE", $outroot); + $inverseSources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $outroot); +} + +my $cmdflags; + +# Perform subtraction +{ + my $command = "$ppSub $outroot"; + $command .= " -inimage $input"; + $command .= " -refimage $template"; + $command .= " -inmask $inputMask"; + $command .= " -refmask $templateMask"; + $command .= " -invariance $inputVariance"; + $command .= " -refvariance $templateVariance"; + $command .= " -insources $inputSources"; + $command .= " -refsources $templateSources"; + $command .= " -stats $outputStats"; + $command .= " -threads $threads" if defined $threads; + if ($run_state eq "new") { + $command .= " -dumpconfig $configuration"; + } else { + my $configurationReal = $ipprc->file_resolve($configuration) or &my_die("Couldn't resolve configuration file: $configuration", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR); + $command .= " -ipprc $configurationReal"; + } + $command .= " -save-inconv" if defined $saveInConv; + $command .= " -save-refconv" if defined $saveRefConv; + $command .= " -recipe PPSUB $recipe_ppSub"; + $command .= " -recipe PSPHOT $recipe_psphot"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; + if ($run_state eq "new") { + $command .= " -photometry"; + } + $command .= " -inverse" if $inverse; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id; + $command .= " -source_id $source_id" if defined $source_id; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppSub: $error_code", $sky_id, $skycell_id, $error_code); + } + + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); + + # measure chip stats + $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $sky_id, $skycell_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + + my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + + if (!$quality) { + &my_die("Couldn't find expected output file: $outputName", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputVariance", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance); + &my_die("Couldn't find expected output file: $outputSources", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + &my_die("Couldn't find expected output file: $jpeg1Name", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1Name); + &my_die("Couldn't find expected output file: $jpeg2Name", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2Name); + if ($inverse) { + &my_die("Couldn't find expected output file: $inverseName", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseName); + &my_die("Couldn't find expected output file: $inverseMask", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseMask); + &my_die("Couldn't find expected output file: $inverseVariance", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseVariance); + &my_die("Couldn't find expected output file: $inverseSources", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseSources); + if ($run_state eq 'new') { + &my_die("Couldn't find expected output file: $configuration", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration); + } + } + } + } else { + print "Not executing: $command\n"; + } +} + +unless ($no_update) { + + # Add the subtraction result + { + my $command = "$difftool -sky_id $sky_id -skycell_id $skycell_id"; + $command .= " -magicked $magicked" if $magicked; + if ($run_state eq 'new') { + $command .= " -adddiffskyfile -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + } else { + $command .= " -tofullskyfile"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = $run_state eq "update" ? + "Unable to perform difftool -adddiffskyfile" : + "Unable to perform difftool -tofullskyfile"; + &my_die("$err_message: $error_code", $sky_id, $skycell_id, $error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $sky_id = shift; # Diff identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $sky_id and defined $skycell_id and not $no_update) { + my $command = "$difftool -sky_id $sky_id -skycell_id $skycell_id -fault $exit_code"; + if ($run_state eq 'new') { + $command .= " -adddiffskyfile"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + } else { + $command .= " -updatediffskyfile"; + } + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: branches/pap/ippScripts/scripts/tdl_generate.pl =================================================================== --- branches/pap/ippScripts/scripts/tdl_generate.pl (revision 27708) +++ branches/pap/ippScripts/scripts/tdl_generate.pl (revision 28003) @@ -13,6 +13,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; #print "\n\n"; -#print "Starting script $0 on $host\n\n"; +#print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -58,5 +59,5 @@ # resolve any path:// or file:// in outroot $outroot = $ipprc->file_resolve($outroot); - + # Look for programs we need my $missing_tools; Index: branches/pap/ippScripts/scripts/warp_overlap.pl =================================================================== --- branches/pap/ippScripts/scripts/warp_overlap.pl (revision 27708) +++ branches/pap/ippScripts/scripts/warp_overlap.pl (revision 28003) @@ -8,6 +8,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); Index: branches/pap/ippScripts/scripts/warp_skycell.pl =================================================================== --- branches/pap/ippScripts/scripts/warp_skycell.pl (revision 27708) +++ branches/pap/ippScripts/scripts/warp_skycell.pl (revision 28003) @@ -7,6 +7,7 @@ use Sys::Hostname; my $host = hostname(); +my $date = `date`; print "\n\n"; -print "Starting script $0 on $host\n\n"; +print "Starting script $0 on $host at $date\n\n"; use DateTime; @@ -299,5 +300,5 @@ $command .= " -tess_id $tess_dir"; $command .= " -path_base $outroot"; # needed for logfile lookups - $command .= " -magicked $magicked" if $magicked; + $command .= " -set_magicked $magicked" if $magicked; $command .= " -uri $outputImage" if !$quality; @@ -321,4 +322,5 @@ $command .= " -warp_id $warp_id"; $command .= " -skycell_id $skycell_id"; + $command .= " -set_magicked $magicked" if $magicked; $command .= " -dbname $dbname" if defined $dbname;