Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/background_chip.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/background_chip.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/background_chip.pl (revision 37068) @@ -34,4 +34,5 @@ my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -134,5 +135,11 @@ } my $in_wt = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $in_path, $class_id); -my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id); +my $in_bg; +if (1) { + $in_bg = $ipprc->filename("PPIMAGE.BACKMDL", $cam_path_base, $class_id); +} +else { + $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id); +} my $in_pattern = $ipprc->filename("PPIMAGE.PATTERN", $in_path, $class_id); my $in_config = $ipprc->filename("PPIMAGE.CONFIG", $in_path, $class_id); @@ -220,4 +227,16 @@ } +my $do_binned_images = 1; # Generate the binned images that are useful for making JPEGs +if ($do_binned_images) { + my $command = "$ppImage -file $out_image -mask $out_mask $outroot -recipe PPIMAGE PPIMAGE_PA -Db PHOTOM F"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $chip_bg_id, $class_id, $error_code); + } +} + + if (!$quality and !$no_op) { &my_die("Couldn't find expected output file: $out_image", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image); Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_advancerun.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_advancerun.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_advancerun.pl (revision 37068) @@ -69,4 +69,5 @@ my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1); +my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1); my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); if ($missing_tools) { @@ -77,4 +78,5 @@ my $tool_cmd; +my $tool_cmd2; my $list_mode; my $component_key; @@ -127,4 +129,9 @@ $list_mode = "-diffskyfile"; $component_key = "skycell_id"; +} elsif ($stage eq "ff") { + $tool_cmd = "$fftool -ff_id"; + $list_mode = "-result"; + $component_key = "dist_component"; + $tool_cmd2 = "$fftool -summary -ff_id" } else { &my_die("Unexpected stage: $stage", $dist_id, $PS_EXIT_CONFIG_ERROR); @@ -132,4 +139,5 @@ $tool_cmd .= " $stage_id"; +$tool_cmd2 .= " $stage_id" if $tool_cmd2; my $exportarg = '-exportrun'; @@ -215,4 +223,26 @@ my $components = parse_md_list($metadata) or &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR); + + if ($tool_cmd2) { + my $command = "$tool_cmd2"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $error_code); + } + if (@$stdout_buf == 0) { + &my_die("Unable to perform $command: $error_code", $dist_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $dist_id, $PS_EXIT_UNKNOWN_ERROR); + my $more_components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR); + if (scalar @$more_components) { + push @$components, @$more_components; + } + } open MANIFEST, ">$resolved" or Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_bundle.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_bundle.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_bundle.pl (revision 37068) @@ -67,6 +67,5 @@ ); my %empty_cleaned = (); - - +my %empty_cleaned = (); # Look for programs we need @@ -496,6 +495,8 @@ } elsif ($stage eq "stack") { $type = $stack_cleaned{$rule}; - } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary') { + } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary' or $stage eq 'ff') { $type = $empty_cleaned{$rule}; + } elsif ($stage eq "skycal") { + $type = $skycal_cleaned{$rule}; } else { &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR); @@ -588,4 +589,11 @@ # stack_summary stage does not use a config dump file. return build_stack_summary_file_list($path_base, \@file_list); + } elsif ($stage eq "ff") { + if ($component eq 'summary') { + # full force summary does not use a config dump file. Cobble together a file list + return build_ff_summary_file_list($path_base, \@file_list); + } else { + $config_file_rule = "PSPHOT.SKY.CONFIG"; + } } else { &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR); @@ -742,4 +750,21 @@ } +# psphotFullForceSummary does not produce a config dump file cobble together a file list +sub build_ff_summary_file_list { + my ($path_base, $file_list) = @_; + + my %cmf ; + $cmf{file_rule} = "SOURCES"; + $cmf{name} = "$path_base.cmf"; + push @$file_list, \%cmf; + + my %log; + $log{file_rule} = "LOG"; + $log{name} = "$path_base.log"; + push @$file_list, \%log; + + return $file_list; +} + sub my_die { Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_defineruns.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_defineruns.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_defineruns.pl (revision 37068) @@ -42,5 +42,5 @@ # Parse the command-line arguments -my ($stage, $stage_limit, $dist_root, $workdir, $no_magic); +my ($stage, $stage_limit, $dist_root, $workdir, $no_magic, $rerun); my ($dbname, $save_temps, $verbose, $no_update, $logfile); my @labels; @@ -51,4 +51,5 @@ 'workdir=s' => \$workdir, # workdir 'stage_limit=s' => \$stage_limit,# maximum number of runs queued for each stage + 'rerun' => \$rerun, # queue new runs even if one exists # 'dist_root=s' => \$dist_root, # root of distribution work area 'no_magic' => \$no_magic, # queue runs without requiring magic (for testing only) @@ -56,4 +57,5 @@ 'verbose' => \$verbose, # Print stuff? 'no-update' => \$no_update, # Don't update the database + 'pretend' => \$no_update, # Don't update the database 'save-temps' => \$save_temps, # Save temporary files? 'logfile=s' => \$logfile, @@ -88,5 +90,5 @@ push @stages, $stage; } else { - @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle); + @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle ff); } @@ -104,4 +106,5 @@ $command .= " -singlefilter" if $single; $command .= " -no_magic" if $no_magic; + $command .= " -rerun" if $rerun; $command .= " -pretend" if $no_update; $command .= " -limit $stage_limit" if $stage_limit; Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_make_fileset.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_make_fileset.pl (revision 37068) @@ -223,5 +223,5 @@ my $dbname = shift; - if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff') or $stage eq 'sky' or $stage eq 'skycal') { + if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff') or $stage eq 'sky' or $stage eq 'skycal' or $stage eq "ff") { return ""; } Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/ipp_apply_burntool_single.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/ipp_apply_burntool_single.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/ipp_apply_burntool_single.pl (revision 37068) @@ -45,4 +45,5 @@ defined $dbname; +# XXX watch out for TC3 and GPC2 here: unless (defined $camera) { if ($this_uri =~ /ota/) { Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/psphot_fullforce_summary.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/psphot_fullforce_summary.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/psphot_fullforce_summary.pl (revision 37068) @@ -0,0 +1,305 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Temp qw( tempfile ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $psphotFullForceSummary = can_run('psphotFullForceSummary') or (warn "Can't find psphotFullForceSummary" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($ff_id, $outroot, $reduction, $camera); +my ($dbname, $threads, $verbose, $save_temps, $no_update, $no_op, $redirect); + +GetOptions( + 'ff_id=s' => \$ff_id, + 'camera=s' => \$camera, # camera name of sources + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --ff_id --outroot --camera", + -exitval => 3, + ) + unless defined $ff_id, + and defined $camera + and defined $outroot; + +my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $ff_id, $PS_EXIT_CONFIG_ERROR ); + +my $neb; +my $scheme = file_scheme($outroot); +if ($scheme and $scheme eq 'neb') { + $neb = $ipprc->nebulous(); +} + +my $logDest = $ipprc->filename("LOG.EXP", $outroot); + +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", + $ff_id, $PS_EXIT_SYS_ERROR ) if $redirect; + + +my ($listFile, $listName) = tempfile("/tmp/fullforce.summary.list.XXXX", UNLINK => !$save_temps ); + +my $cff_file; + +{ + my $mdcParser = PS::IPP::Metadata::Config->new; + my $results; + { + my $command = "$fftool -result -ff_id $ff_id -quality 0"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform fftool -result $error_code", + $ff_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $ff_id, $PS_EXIT_PROG_ERROR); + $results = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $ff_id, $PS_EXIT_PROG_ERROR); + } + + &my_die("result list is empty.", $ff_id, $PS_EXIT_SYS_ERROR) + if scalar @$results == 0; + + print $listFile "SOURCES MULTI\n"; + foreach my $result (@$results) { + my $cmf = $ipprc->filename('PSPHOT.FULLFORCE.OUTPUT', $result->{path_base}); + &my_die("Couldn't find input cmf: $cmf", $ff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($cmf); + print $listFile "SOURCES STR $cmf\n"; + if (!$cff_file) { + $cff_file = $ipprc->filename('PSPHOT.OUTPUT.CFF', $result->{sources_path_base}); + &my_die("Couldn't find input cff: $cff_file", $ff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($cff_file); + } + } + close $listFile; +} + +&my_die("No CFF found in results: $cff_file", $ff_id, $PS_EXIT_PROG_ERROR) unless $cff_file; + + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_psphot = $ipprc->reduction($reduction, 'FULLFORCE_PSPHOT'); # Recipe to use for psphot +unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for PSPHOT: $reduction\n", + $ff_id, $PS_EXIT_CONFIG_ERROR); +} + +# XXX: need to figure out whether this works or not +# We probably want to create a specific recipe that looks at the results +my $recipe_ppstats = 'WARPSTATS'; +my $doStats = 0; + +print "reduction: $reduction\n"; +print "recipe_psphot: $recipe_psphot\n"; + +my $dump_config = 1; + +# Get the output filenames +my $outputSources = prepare_output("PSPHOT.FULLFORCE.OUTPUT", $outroot, 1); +my $configuration = prepare_output("PSPHOT.SKY.CONFIG", $outroot, 1) if $dump_config; +my $outputStats = prepare_output("SKYCELL.STATS", $outroot, 1) if $doStats; +my $traceDest = prepare_output("TRACE.EXP", $outroot, 1); + +my $cmdflags = ""; + +# Perform psphotFullForceSummary +{ + my $command = "$psphotFullForceSummary $outroot"; + $command .= " -input $listName"; + $command .= " -cff $cff_file"; + $command .= " -threads $threads" if defined $threads; + if ($dump_config) { + $command .= " -dumpconfig $configuration"; + } + $command .= " -recipe PSPHOT $recipe_psphot"; + if ($doStats) { + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS $recipe_ppstats"; + } +# $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psphotFullForceSummary: $error_code", $ff_id, $error_code); + } + + # Stats: TODO + if ($doStats) { + check_output($outputStats, 1); + my $outputStatsReal = $ipprc->file_resolve($outputStats); + + # measure chip stats + $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $ff_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + } + my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + + if (!$quality) { + check_output($outputSources, 1); + } + } else { + print "Not executing: $command\n"; + } +} + +# Add the result to the database +{ + my $command = "$fftool -ff_id $ff_id"; + $command .= " -addsummary -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = "Unable to perform fftool -addwarped" ; + &my_die("$err_message: $error_code", $ff_id, $error_code); + } + } else { + print "Not executing $command\n"; + } +} + +exit 0; + + +# Prepare to write to an output file +# Lookup the filename in the rules. +# Make sure that if file exists and is a nebulous file that there is only one instance +# Deal with files that have been lost. +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, undef, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $ff_id, $error); + return $output; +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $ff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $ff_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $ff_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + + if ($replicate and $neb) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $ff_id, $PS_EXIT_SYS_ERROR); + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $ff_id = shift; # full force run identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $ff_id) { + my $command = "$fftool -ff_id $ff_id -fault $exit_code"; + $command .= " -addsummary"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + run(command => $command, verbose => $verbose); + } else { + print "not executing $command\n"; + } + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/psphot_fullforce_warp.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/psphot_fullforce_warp.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/psphot_fullforce_warp.pl (revision 37068) @@ -0,0 +1,333 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $psphotFullForce = can_run('psphotFullForce') or (warn "Can't find psphotFullForce" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($ff_id, $warp_id, $skycell_id, $path_base, $sourceroot, $camera); +my ($outroot, $reduction); +my ($dbname, $threads, $verbose, $no_update, $no_op, $redirect); + +GetOptions( + 'ff_id=s' => \$ff_id, + 'warp_id=s' => \$warp_id, # warp identifier + 'skycell_id=s' => \$skycell_id,# Skycell identifier + 'warp_path_base=s' => \$path_base, # path_base of the warp skycell + 'sources_path_base=s' => \$sourceroot,# path_base of sources + 'camera=s' => \$camera, # camera name of sources + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --ff_id --warp_id --sourceroot --skycell_id --warp_path_base --outroot --camera", + -exitval => 3, + ) unless defined $ff_id, + and defined $sourceroot + and defined $path_base + and defined $warp_id + and defined $skycell_id + and defined $camera + and defined $outroot; + +my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $ff_id, $warp_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); + +my $neb; +my $scheme = file_scheme($outroot); +if ($scheme and $scheme eq 'neb') { + $neb = $ipprc->nebulous(); +} + +my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id); + +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", + $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +if (0) { +# if (!$path_base) { + # If path_base is not supplied, look it up in the database. + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $files; + { + my $command = "$warptool -warped -warp_id $warp_id -skycell_id $skycell_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -warpskyfile -inputskyfile: $error_code", + $ff_id, $warp_id, $skycell_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $ff_id, $warp_id, $skycell_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $ff_id, $warp_id, $skycell_id, $PS_EXIT_PROG_ERROR); + } + + &my_die("Input list does not contain exactly one elements", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) + unless scalar @$files == 1; + + my $warp = $files->[0]; + + $path_base = $warp->{path_base}; + &my_die("Couldn't find input path in warptool output", $ff_id, $warp_id, $skycell_id, $PS_EXIT_UNKNOWN_ERROR) + unless defined $path_base; + +} + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_psphot = $ipprc->reduction($reduction, 'FULLFORCE_PSPHOT'); # Recipe to use for psphot +unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for PSPHOT: $reduction\n", + $ff_id, $warp_id, $skycell_id, $PS_EXIT_CONFIG_ERROR); +} + +# XXX: need to figure out whether this works or not +# We probably want to create a specific recipe that looks at the results +my $recipe_ppstats = 'WARPSTATS'; +my $doStats = 0; + +print "reduction: $reduction\n"; +print "recipe_psphot: $recipe_psphot\n"; + +# use psf measured on input warp +# XXX: get this from recipe +my $useWarpPSF = 0; + +my $input = $ipprc->filename('PSWARP.OUTPUT', $path_base); +my $inputMask = $ipprc->filename('PSWARP.OUTPUT.MASK', $path_base); +my $inputVariance = $ipprc->filename('PSWARP.OUTPUT.VARIANCE', $path_base); +my $inputPSF = $useWarpPSF ? $ipprc->filename('PSPHOT.PSF.SKY.SAVE', $path_base) : ""; +my $inputSources = $ipprc->filename('PSPHOT.OUTPUT.CFF', $sourceroot); + +if ($verbose) { + print "input: $input\n"; + print "inputMask: $inputMask\n"; + print "inputVariance: $inputVariance\n"; + print "inputPSF: $inputPSF\n" if $inputPSF; + print "inputSources: $inputSources\n"; +} + +# check that the inputs exist (and have non-zero size) +&my_die("Couldn't find input: $input", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input); +&my_die("Couldn't find input: $inputMask", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask); +&my_die("Couldn't find input: $inputVariance", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputVariance); +&my_die("Couldn't find input: $inputPSF", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) if ($inputPSF && !$ipprc->file_exists($inputPSF)); +&my_die("Couldn't find input: $inputSources", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputSources); + +my $dump_config = 1; + +# Get the output filenames +my $outputSources = prepare_output("PSPHOT.OUT.CMF.MEF", $outroot, 1); +my $configuration = prepare_output("PSPHOT.SKY.CONFIG", $outroot, 1) if $dump_config; +my $outputStats = prepare_output("SKYCELL.STATS", $outroot, 1) if $doStats; +my $traceDest = prepare_output("TRACE.EXP", $outroot, 1); + +my $cmdflags = ""; + +# Perform psphotFullForce +{ + my $command = "$psphotFullForce $outroot"; + $command .= " -force $inputSources"; + $command .= " -file $input"; + $command .= " -mask $inputMask"; + $command .= " -variance $inputVariance"; + $command .= " -psf $inputPSF" if $inputPSF; + $command .= " -threads $threads" if defined $threads; + if ($dump_config) { + $command .= " -dumpconfig $configuration"; + } + $command .= " -recipe PSPHOT $recipe_psphot"; + if ($doStats) { + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS $recipe_ppstats"; + } + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; + $command .= " -photcode-rule '{DETECTOR}.{FILTER.ID}.ForcedWarp'"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psphotFullForce: $error_code", $ff_id, $warp_id, $skycell_id, $error_code); + } + + # Stats: TODO + if ($doStats) { + check_output($outputStats, 1); + my $outputStatsReal = $ipprc->file_resolve($outputStats); + + # measure chip stats + $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $ff_id, $warp_id, $skycell_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + } + my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + + if (!$quality) { + check_output($outputSources, 1); + } + } else { + print "Not executing: $command\n"; + } +} + +# Add the result to the database +{ + my $command = "$fftool -ff_id $ff_id -warp_id $warp_id"; + $command .= " -addresult -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = "Unable to perform fftool -addwarped" ; + &my_die("$err_message: $error_code", $ff_id, $warp_id, $skycell_id, $error_code); + } + } else { + print "Not executing $command\n"; + } +} + +exit 0; + + +# Prepare to write to an output file +# Lookup the filename in the rules. +# Make sure that if file exists and is a nebulous file that there is only one instance +# Deal with files that have been lost. +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, undef, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $ff_id, $warp_id, $skycell_id, $error); + return $output; +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + + if ($replicate and $neb) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $ff_id = shift; # full force run identifier + my $warp_id = shift; # full force warp id + my $skycell_id = shift; # Skycell identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $ff_id and defined $skycell_id) { + my $command = "$fftool -ff_id $ff_id -warp_id $warp_id -fault $exit_code"; + $command .= " -addresult"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + run(command => $command, verbose => $verbose); + } else { + print "not executing $command\n"; + } + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/queuestaticsky.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/queuestaticsky.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/queuestaticsky.pl (revision 37068) @@ -35,4 +35,6 @@ }; +# don't update the database if we are pretending +$no_update = 1 if $pretend; my $missing_tools; @@ -116,4 +118,5 @@ } } + last if $pretend; # only do the first (full) set if we are in pretend mode } { Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/receive_advance.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/receive_advance.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/receive_advance.pl (revision 37068) @@ -65,4 +65,5 @@ my ($stage) = $filename =~ m|^dbinfo\.(\S+)\.\d+\.mdc$|; # Stage of interest my $tool_name; + my $tool_mode = '-importrun'; if ($stage eq "raw") { $tool_name = "regtool"; @@ -73,4 +74,7 @@ } elsif ($stage eq "sky") { $tool_name = "staticskytool"; + } elsif ($stage eq "skycal") { + $tool_name = "staticskytool"; + $tool_mode = '-importskycal'; } else { $tool_name = "${stage}tool"; @@ -81,5 +85,5 @@ &my_die("Unable to resolve $dbinfo_uri\n", $PS_EXIT_UNKNOWN_ERROR) unless $file; - my $command = "$tool -importrun -infile $file"; # Command to execute + my $command = "$tool $tool_mode -infile $file"; # Command to execute $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/receive_file.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/receive_file.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/receive_file.pl (revision 37068) @@ -223,7 +223,8 @@ } elsif ($runType eq 'staticskyRun') { $stage = 'sky'; - # XXX: This should be skycell, but the distribution code uses exposure $comp_name = 'skycell_id'; -# $current_component = $comp_name; + } elsif ($runType eq 'skycalRun') { + $stage = 'skycal'; + $comp_name = 'skycell_id'; } else { &my_die( "unexpected run type line found in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR); @@ -238,9 +239,10 @@ } - if ($stage eq 'sky') { - # the dbinfo file for a skyRun only has one component and it doesn't contain + if ($stage eq 'sky' or $stage eq 'skycal') { + # the dbinfo file for staticskyRun and skycalRun only have one component and they don't contain # skycell_id which is the way components are listed in the dirinfo file. my @ids = keys %$components; - &my_die( "unexpected number of components scalar @ids found in staticsky dirinfo file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if scalar @ids != 1; + my $nComponents = scalar @ids; + &my_die( "unexpected number of components $nComponents found in $stage dirinfo file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if $nComponents != 1; $current_component = $ids[0]; } Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_mk_stack_mdc.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_mk_stack_mdc.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_mk_stack_mdc.pl (revision 37068) @@ -0,0 +1,19 @@ +#! /usr/bin/env perl + +use Carp; +use warnings; +use strict; + +my $counter = 0; + +foreach my $path_base (@ARGV) { + print "INPUT${counter} METADATA\n"; + print " IMAGE STR ${path_base}.fits\n"; + print " MASK STR ${path_base}.mask.fits\n"; + print " VARIANCE STR ${path_base}.wt.fits\n"; + print " PSF STR ${path_base}.psf\n"; + print " SOURCES STR ${path_base}.cmf\n"; + print " BKGMODEL STR ${path_base}.mdl.fits\n"; + print "END\n\n"; + $counter++; +} Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_camera.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_camera.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_camera.pl (revision 37068) @@ -0,0 +1,362 @@ +#! /usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use DateTime; +use Data::Dumper; +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Hard coded values +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $threads = 2; # How many threads are we going to use? +my $job_cost = 1700 / 60 / 60; # Estimate of how long a job runs, in hours. +my $proc_per_node = 24; # processors per node +my $min_nodes = 1; # smallest allocation to ask for +my $max_nodes = 1000; # largest allocation to ask for +my $min_time = 1; # shortest allocation to ask for +my $max_time = 8; # longest allocation to ask for +my $remote_hostname= "LANL/Mustang"; # Name of the remote node. +my @return_component_list = ("DBINFO.EXP", "PSASTRO.CONFIG", "PSASTRO.OUTPUT", "LOG.EXP","TRACE.EXP", "PSASTRO.STATS"); +# Look for programs we need +my $missing_tools; + +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($remote_id,$cam_id,,$camera,$dbname,$verbose,$path_base,$no_update); +GetOptions( + 'remote_id=s' => \$remote_id, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, + 'path_base=s' => \$path_base, + 'no_update' => \$no_update, + 'verbose' => \$verbose, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless + defined($remote_id) and + defined($camera) and + defined($path_base) and + defined($dbname); +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# +# Step 1: Get a list of the components that make up this remoteRun + +# SHould this call listrun to ensure we're in state new? +my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; +$rt_cmd .= " -dbname $dbname " if defined($dbname); + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $rt_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run remotetool to determine stage parameters.", + $remote_id,$error_code); +} +my $compData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine component information.", + $remote_id,$PS_EXIT_PROG_ERROR); +my $compData2= parse_md_list($compData); + +# +# Step 1b: Open output files +my $uri_command = $path_base . ".cmd"; +my $uri_transfer= $path_base . ".transfer"; +my $uri_check = $path_base . ".check"; +my $uri_config = $path_base . ".config"; +my $uri_generate= $path_base . ".generate"; +my $uri_return = $path_base . ".return"; + +my $disk_command = $ipprc->file_resolve($uri_command,1); +my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); +my $disk_check = $ipprc->file_resolve($uri_check,1); +my $disk_config = $ipprc->file_resolve($uri_config,1); +my $disk_generate= $ipprc->file_resolve($uri_generate,1); +my $disk_return = $ipprc->file_resolve($uri_return,1); + +my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id,$PS_EXIT_SYS_ERROR); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$remote_id,$PS_EXIT_SYS_ERROR); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$remote_id,$PS_EXIT_SYS_ERROR); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id,$PS_EXIT_SYS_ERROR); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $PS_EXIT_SYS_ERROR); +my %file_filter = (); + +# +# Step 2: Iterate over all componenets in this remote run. +my $job_index = 0; +my @pre_commands = (); +my @main_commands = (); +my @post_commands = (); + +foreach my $compEntry (@$compData2) { + my $cam_id = $compEntry->{stage_id}; + +# Get exposure level information from the camRun we're working from. + + my $command = "$camtool -pendingexp -cam_id $cam_id "; + $command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run camtool to determine stage parameters.", + $cam_id,$error_code); + } + + my $camData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine cam component information.", + $cam_id,$PS_EXIT_PROG_ERROR); + my $camData2= parse_md_list($camData); + my $camEntry = ${ $camData2 }[0]; + + my $workdir = $camEntry->{workdir}; + my $exp_tag = $camEntry->{exp_tag}; + unless (defined($workdir)) { + while( my ($k, $v) = each %$camEntry ) { + print "key: $k, value: $v.\n"; + } + + print "%{ $camEntry }\n"; + die; + } + my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.cm.${cam_id}"; + my $remote_outroot = uri_local_to_remote($ipp_outroot); + + +# +# Step 3: Iterate over the sub-components + + my $reg_command = "$camtool -pendingimfile -cam_id $cam_id "; + $reg_command .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $reg_command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run camtool -pendingimfile ", + $cam_id,$error_code); + } + + + # We don't actually care about the input cam data other than to know which mask files to instantiate. + my $inpData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine cam component information.", + $cam_id, $PS_EXIT_PROG_ERROR); + my $inpData2=parse_md_list($inpData); + my $chipProto= ${ $inpData2 }[0]; + my $chip_path = $chipProto->{path_base}; + my $remote_chip_path = uri_local_to_remote( $chip_path ); + my $pre_cmd_cmfs = "ls -1 ${remote_chip_path}*.cmf > ${remote_outroot}.cmflist"; + my $pre_cmd_masks= "ls -1 ${remote_chip_path}*.mk.fits > ${remote_outroot}.masklist"; + + my $reduction = 'DEFAULT' unless defined $camEntry->{reduction}; + my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use + + my $psastro_command = " psastro -list ${remote_outroot}.cmflist "; + $psastro_command .= " -masklist ${remote_outroot}.masklist ${remote_outroot} "; +# $psastro_command .= " -refmasklist ${remote_outroot}.masklist ${remote_outroot} "; # This is used to do edge calculations. It should probably be smart enough to know that it can just use the masks I just specified, but it's not. + # Uncomment when remote psastro supports this + $psastro_command .= " -kh-correct /turquoise/usr/projects/ps1/watersc1/references/khcorrect.20140606.v0.fits "; + $psastro_command .= " -refmasklist /turquoise/usr/projects/ps1/watersc1/references/gpc1.refmask.list "; +# $psastro_command .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20080909.asm "; + $psastro_command .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20140505.asm "; + $psastro_command .= " -recipe PSASTRO $recipe_psastro "; + $psastro_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; + $psastro_command .= " -dumpconfig ${remote_outroot}.mdc -stats ${remote_outroot}.stats "; + $psastro_command .= " -recipe PPSTATS CAMSTATS "; + + my $camtool_post_cmd = "camtool -cam_id $cam_id -addprocessedexp -uri UNKNOWN "; + $camtool_post_cmd .= " -dbname $dbname " if defined $dbname; $camtool_post_cmd .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 "; + my $post_cmd_echo = " echo -n \"$camtool_post_cmd\" > ${remote_outroot}.dbinfo "; + my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - CAMERA_EXP_FPA >> ${remote_outroot}.dbinfo "; + + print CONFIG "${pre_cmd_cmfs} && ${pre_cmd_masks} && ${psastro_command} && ${post_cmd_echo} && ${post_cmd_SfM} "; + $job_index++; + + # Determine which output files need to be returned + foreach my $component(@return_component_list) { + my $filename = $ipprc->filename($component,$ipp_outroot); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} "; + } + foreach my $chipInfo (@{ $inpData2 }) { + my $filename = $ipprc->filename("PSASTRO.OUTPUT.MASK",$ipp_outroot,$chipInfo->{class_id}); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} "; + } + print CONFIG "\n"; +} +close(CONFIG); +close(TRANSFER); +close(CHECK); +close(RETURN); +close(GENERATE); + +# +# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. +my $proc_need = $job_index * $threads; # how many total processors do we need? +my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? +my $time_need = $job_index * $job_cost; # How many seconds will this take? + +my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill +my ($time_req,$node_req); +if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { + $time_req = $min_time; + $node_req = $min_nodes; +} +elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { + $time_req = $max_time; + $node_req = $max_nodes; + print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; +} +else { + $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; + $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; +} + +open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR); +print COMMAND "#!/bin/tcsh\n"; +print COMMAND "##### Moab controll lines\n"; +print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES +print COMMAND "#MSUB -j oe\n"; +print COMMAND "#MSUB -V\n"; +print COMMAND "#MSUB -o ${remote_root}cam.${remote_id}.out\n"; +print COMMAND "date\n"; +print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; +print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; +print COMMAND "date\n"; +close(COMMAND); + +unless($no_update) { + my $command = "remotetool -updaterun -remote_id $remote_id "; + $command .= " -set_state pending "; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); +} + + +## Common SC routines + +sub uri_convert { # (ipp_disk,remote_disk) = uri_convert(neb_uri); + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_convert_and_create { # (ipp_disk,remote_disk) = uri_convert_and_create(neb_uri); ipp_disk is created if it doesn't exist + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_local_to_remote { #(remote_uri) = uri_local_to_remote(local_neb_uri); + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); + # This needs to replace the remote root directory with the nebulous tag. + my $remote_uri = shift; + $remote_uri =~ s%${remote_root}%%; + my $local_uri = "neb:///" . $remote_uri; + + return($local_uri); +} + + +sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status); + my $msg = shift; + my $id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $id and not $no_update) { + my $command = "remotetool -updaterun -remote_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_chip.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_chip.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_chip.pl (revision 37068) @@ -0,0 +1,547 @@ +#! /usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use DateTime; +use Data::Dumper; +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Hard coded values +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. +my $threads = 2; # How many threads are we going to use? +my $job_cost = 150 / 60 / 60; # Estimate of how long a job runs, in hours. +my $proc_per_node = 24; # processors per node +my $min_nodes = 1; # smallest allocation to ask for +my $max_nodes = 1000; # largest allocation to ask for +my $min_time = 1; # shortest allocation to ask for +my $max_time = 8; # longest allocation to ask for + +# We need to ensure we only ever try to transfer a file once. +my %file_filter = (); + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $detselect= can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Options +my ($exp_id,$exp_name,$remote_id,$chip_id,$detrends,$camera,$dbname,$verbose,$path_base,$no_update); +GetOptions( + 'remote_id=s' => \$remote_id, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, + 'path_base=s' => \$path_base, + 'no_update' => \$no_update, + 'verbose' => \$verbose, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless + defined($remote_id) and + defined($camera) and + defined($path_base) and + defined($dbname); +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# +# Step 1: Get a list of the components that make up this remoteRun + +# SHould this call listrun to ensure we're in state new? +my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; +$rt_cmd .= " -dbname $dbname " if defined($dbname); + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $rt_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool to determine stage parameters.", + $remote_id,$error_code); +} +my $compData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine component information.", + $remote_id,$PS_EXIT_PROG_ERROR); +my $compData2= parse_md_list($compData); + +# +# Step 1a: Load up the catalog of detrends +# Detrend concept holders +my %detrends = (); +my %det_types = ('MASK' => '-mask', + 'FLAT' => '-flat', + 'DARK' => '-dark', + 'VIDEODARK' => '-dark', + 'LINEARITY' => '-linearity', + 'FRINGE' => '-fringe', + 'NOISEMAP' => '-noisemap'); + +foreach my $det_type (keys (%det_types)) { + my $dt_cmd = "$detselect -show -camera $camera -det_type $det_type "; + $dt_cmd .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $dt_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("UNable to run detselect to determine detrend catalog", + $remote_id,$error_code); + } + my $detData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine detrend information.", + $remote_id,$PS_EXIT_PROG_ERROR); + my $detData2= parse_md_list($detData); + + # Data structure for this: + # $detrend{NAME} = \@list_of_detselect_shows + # $detrend{NAME}[0] = \%hash_of_detselect_constraint_results (first element) + # $detrend{NAME}[0]{class_id} = uri + + # At this point, I have the list of detselect shows, assign it + $detrends{$det_type} = $detData2; + + for (my $dd = 0; $dd <= $#{ @{ $detrends{$det_type} } }; $dd++) { + my $det_id = ${ $detrends{$det_type} }[$dd]->{det_id}; + my $det_iter=${ $detrends{$det_type} }[$dd]->{iteration}; + + my $detselect_command2 = "detselect -select "; + $detselect_command2 .= " -det_id $det_id "; + $detselect_command2 .= " -iteration $det_iter "; + $detselect_command2 .= " -dbname $dbname " if defined($dbname); + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $detselect_command2, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", + $chip_id,$error_code); + } + + my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", + $chip_id,$error_code); + my $detImfile2= parse_md_list($detImfile); + foreach $detImfile (@$detImfile2) { + my $class_id = $detImfile->{class_id}; + my $duri = $detImfile->{uri}; + + ${ $detrends{$det_type} }[$dd]->{$class_id} = $duri; + } + } +} + +#print Dumper(%detrends); + +my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","TRACE.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN"); + +# +# Step 1b: Open output files +my $uri_command = $path_base . ".cmd"; +my $uri_transfer= $path_base . ".transfer"; +my $uri_check = $path_base . ".check"; +my $uri_config = $path_base . ".config"; +my $uri_generate= $path_base . ".generate"; +my $uri_return = $path_base . ".return"; + +my $disk_command = $ipprc->file_resolve($uri_command,1); +my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); +my $disk_check = $ipprc->file_resolve($uri_check,1); +my $disk_config = $ipprc->file_resolve($uri_config,1); +my $disk_generate= $ipprc->file_resolve($uri_generate,1); +my $disk_return = $ipprc->file_resolve($uri_return,1); + +my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$chip_id,$PS_EXIT_SYS_ERROR); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR); + + +# +# Step 2: Iterate over all componenets in this remote run. +my $job_index = 0; +my @pre_commands = (); +my @main_commands = (); +my @post_commands = (); + +foreach my $compEntry (@$compData2) { + + my $chip_id = $compEntry->{stage_id}; + my $exp_id ; +# Get exposure level information from the chipRun we're working from. + my $command = "$chiptool -listrun -chip_id $chip_id "; + $command .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool to determine stage parameters.", + $chip_id,$error_code); + } + +# Parse chipRun level data to determine detrend information + my %detrends_to_use = (); + { + my $chipData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine chip component information.", + $chip_id,$PS_EXIT_PROG_ERROR); + my $chipData2= parse_md_list($chipData); + my $chipEntry = ${ $chipData2 }[0]; + + $exp_id = $chipEntry->{exp_id}; + my $filter = $chipEntry->{filter}; + my $altfilt= $filter; + $altfilt =~ s/.00000//; + my $dateobs= $chipEntry->{dateobs}; + my ($date, $time) = split /T/, $dateobs; + my ($year,$month,$day) = split /-/, $date; + my ($hour,$minute,$second) = split /:/, $time; #/; + my $dateobs_obj = DateTime->new(year => $year, month => $month, day => $day, hour => $hour, minute => $minute, second => $second); + + foreach my $det_type (%detrends) { + if (($filter !~ /y/)&&($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y + + foreach my $det_obj (@{ $detrends{$det_type} }) { # Iterate over the available options + if (defined($det_obj->{filter})) { + if (($det_obj->{filter} ne $filter)&&($det_obj->{filter} ne $altfilt)) { + next; # Skip detrends with a filter that we can't use. + } + } + if (defined($det_obj->{time_begin})) { + if (DateTime->compare($dateobs_obj,$det_obj->{time_begin}) == -1) { + next; # Skip detrends that come into effect after this exposure (d1 < d2) + } + } + if (defined($det_obj->{time_end})) { + if (DateTime->compare($dateobs_obj,$det_obj->{time_end}) == 1) { + next; # Skip detrends that stop working before this exposure (d1 > d2) + } + } + $detrends_to_use{$det_type} = $det_obj; + last; + } + } + } + +# +# Step 3: Iterate over the sub-components +# Iterate over the chipProcessedImfile level data. + my $reg_command = "$chiptool -pendingimfile "; + $reg_command .= " -chip_id $chip_id " if defined($chip_id); + $reg_command .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $reg_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool -pendingimfile", + $chip_id,$error_code); + } + + my $chipData = $mdcParser->parse(join "", @$stdout_buf) or + next; +# &my_die("Unable to parse chiptool -pendingimfile information.", +# $chip_id,$error_code); + my $chipData2= parse_md_list($chipData); + + + foreach my $chipEntry (@$chipData2) { + # Get information we need to pass to ppImage + my $uri = $chipEntry->{uri}; + my $class_id = $chipEntry->{class_id}; + my $video_cells = $chipEntry->{video_cells}; + my $reduction = $chipEntry->{reduction}; + my $exp_tag = $chipEntry->{exp_tag}; + my $workdir = $chipEntry->{workdir}; + my $chip_imfile_id = $chipEntry->{chip_imfile_id}; + + # Process the image and burntool table + my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri); + my $btt = $uri; + $btt =~ s/fits$/burn.tbl/; + my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt); + + # Initialize the ppI command + my $ppImage_command = "ppImage -file $remote_uri"; + $ppImage_command .= " -burntool $remote_btt "; + + # Add detrend information to the command line + foreach my $det (keys %detrends_to_use) { + if ((($video_cells)&&($det eq 'DARK'))|| + ((!$video_cells)&&($det eq 'VIDEODARK'))) { + next; + } + my $det_code = $det_types{$det}; + my $duri = $detrends_to_use{$det}->{$class_id}; + my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri); + $ppImage_command .= " $det_code $remote_det_uri "; + } + + + # Add output root + my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}"; + my $remote_outroot = uri_local_to_remote($ipp_outroot); + $ppImage_command .= " $remote_outroot "; + print STDERR "$remote_outroot $ipp_outroot $class_id\n"; + # Complete reduction information. + $reduction = 'DEFAULT' unless defined $reduction; + my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage + my $recipe_psphot = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot + + $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage "; + $ppImage_command .= " -recipe PSPHOT $recipe_psphot "; + $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; + $ppImage_command .= " -threads 4 "; + $ppImage_command .= " -image_id $chip_imfile_id "; + $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; + $ppImage_command .= " -log ${remote_outroot}.${class_id}.log "; + +# push @main_commands, $ppImage_command; + + # Calculate pre and post commands + my $remote_outroot_dir = dirname($remote_outroot); + my $pre_command = "mkdir -p $remote_outroot_dir"; + + my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; + $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; + $post_commandA .= " -magicked 0 -hostname REMOTE -dtime_script 0 "; + $post_commandA .= " -dbname $dbname " if defined $dbname; + + my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo "; + my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo"; + +# push @post_commands, "$post_commandB $post_commandC"; + + $job_index++; + + print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}"; + + + # Determine which output files need to be returned. + foreach my $component (@return_component_list) { +# uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) ); + my $filename = $ipprc->filename($component, $ipp_outroot, $class_id); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename ); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} "; +# print " $filename $ipp_disk $remote_disk\n"; +# die; + } + + print CONFIG "\n"; + } +} + +# Actually put the commands into the output file. +# my %unique_cmds = (); +# foreach my $cmd (@pre_commands) { +# unless(exists($unique_cmds{$cmd})) { +# print CONFIG $cmd . "\n"; +# $unique_cmds{$cmd} = 1; +# } +# } +# foreach my $cmd (@main_commands) { +# unless(exists($unique_cmds{$cmd})) { +# print CONFIG $cmd . "\n"; +# $unique_cmds{$cmd} = 1; +# } +# } +# foreach my $cmd (@post_commands) { +# unless(exists($unique_cmds{$cmd})) { +# print CONFIG $cmd . "\n"; +# $unique_cmds{$cmd} = 1; +# } +# } + +close(CONFIG); +close(TRANSFER); +close(CHECK); +close(RETURN); +close(GENERATE); + +# +# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. +my $proc_need = $job_index * $threads; # how many total processors do we need? +my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? +my $time_need = $job_index * $job_cost; # How many seconds will this take? + +my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill +my ($time_req,$node_req); +if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { + $time_req = $min_time; + $node_req = $min_nodes; +} +elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { + $time_req = $max_time; + $node_req = $max_nodes; + print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; +} +else { + $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; + $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; +} + +open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$chip_id,$PS_EXIT_SYS_ERROR); +print COMMAND "#!/bin/tcsh\n"; +print COMMAND "##### Moab controll lines\n"; +print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES +print COMMAND "#MSUB -j oe\n"; +print COMMAND "#MSUB -V\n"; +print COMMAND "#MSUB -o ${remote_root}chip.${remote_id}.out\n"; +print COMMAND "date\n"; +print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; +print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; +#print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n"; +print COMMAND "date\n"; +close(COMMAND); + + +## We're done here. The execution and handling are done elsewhere. +# Quick review: +# new -> pending -> run -> full +# auth +unless($no_update) { + my $command = "remotetool -updaterun -remote_id $remote_id "; + $command .= " -set_state pending "; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); +} + + + + + + +sub uri_convert { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_convert_and_create { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs_raw { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + $remote_disk = $remote_raw . $ipp_disk; + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs_for_return { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_local_to_remote { + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub uri_remote_to_local { + # This needs to replace the remote root directory with the nebulous tag. + my $remote_uri = shift; + $remote_uri =~ s%${remote_root}%%; + my $local_uri = "neb:///" . $remote_uri; + + return($local_uri); +} +sub my_die { + my $msg = shift; + my $id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $id and not $no_update) { + my $command = "remotetool -updaterun -remote_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} + Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_stack.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_stack.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_stack.pl (revision 37068) @@ -0,0 +1,368 @@ +#! /usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use DateTime; +use Data::Dumper; +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Hard coded values +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $threads = 2; # How many threads are we going to use? +my $job_cost = 1500 / 60 / 60; # Estimate of how long a job runs, in hours. +my $proc_per_node = 24; # processors per node +my $min_nodes = 1; # smallest allocation to ask for +my $max_nodes = 1000; # largest allocation to ask for +my $min_time = 1; # shortest allocation to ask for +my $max_time = 8; # longest allocation to ask for +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my @return_component_list = ("DBINFO.EXP", "PPSTACK.UNCONV","PPSTACK.UNCONV.MASK","PPSTACK.UNCONV.VARIANCE", + "PPSTACK.UNCONV.EXP","PPSTACK.UNCONV.EXPWT","PPSTACK.UNCONV.EXPNUM","PSPHOT.OUTPUT", + "LOG.EXP","TRACE.EXP", "PSASTRO.STATS", + "PPSTACK.CONFIG","PPSTACK.TARGET.PSF"); +#,"PPSTACK.CONV.KERNEL"); +# Look for programs we need +my $missing_tools; + +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find warptool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($remote_id,$stack_id,$camera,$dbname,$verbose,$path_base,$no_update); +GetOptions( + 'remote_id=s' => \$remote_id, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, + 'path_base=s' => \$path_base, + 'no_update' => \$no_update, + 'verbose' => \$verbose, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless + defined($remote_id) and + defined($camera) and + defined($path_base) and + defined($dbname); +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# +# Step 1: Get a list of the components that make up this remoteRun + +# SHould this call listrun to ensure we're in state new? +my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; +$rt_cmd .= " -dbname $dbname " if defined($dbname); + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $rt_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run remotetool to determine stage parameters.", + $remote_id,$error_code); +} +my $compData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine component information.", + $remote_id,$PS_EXIT_PROG_ERROR); +my $compData2= parse_md_list($compData); + +# +# Step 1b: Open output files +my $uri_command = $path_base . ".cmd"; +my $uri_transfer= $path_base . ".transfer"; +my $uri_check = $path_base . ".check"; +my $uri_config = $path_base . ".config"; +my $uri_generate= $path_base . ".generate"; +my $uri_return = $path_base . ".return"; + +my $disk_command = $ipprc->file_resolve($uri_command,1); +my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); +my $disk_check = $ipprc->file_resolve($uri_check,1); +my $disk_config = $ipprc->file_resolve($uri_config,1); +my $disk_generate= $ipprc->file_resolve($uri_generate,1); +my $disk_return = $ipprc->file_resolve($uri_return,1); + +my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id,$PS_EXIT_SYS_ERROR); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$remote_id,$PS_EXIT_SYS_ERROR); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$remote_id,$PS_EXIT_SYS_ERROR); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id,$PS_EXIT_SYS_ERROR); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $PS_EXIT_SYS_ERROR); + + +# +# Step 2: Iterate over all componenets in this remote run. +my $job_index = 0; +my @pre_commands = (); +my @main_commands = (); +my @post_commands = (); +my %file_filter = (); + + +foreach my $compEntry (@$compData2) { + my $stack_id = $compEntry->{stage_id}; + +# Get exposure level information from the stackRun we're working from. + my $run_command = "$stacktool -tosum -stack_id $stack_id"; + $run_command .= " -dbname $dbname " if defined($dbname); + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $run_command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run stacktool -tosum ", + $stack_id,$error_code); + } + + my $stackData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine stack component information.", + $stack_id,$PS_EXIT_PROG_ERROR); + my $stackData2= parse_md_list($stackData); + + my $stack = ${ $stackData2 }[0]; + my $reduction = $stack->{reduction}; + $reduction = 'DEFAULT' unless defined($reduction); + my $workdir = $stack->{workdir}; + my $tess_id = $stack->{tess_id}; + my $skycell_id= $stack->{skycell_id}; + + my $ipp_outroot = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.stk.${stack_id}"; + my $remote_outroot = uri_local_to_remote($ipp_outroot); + my $remote_outdir = uri_local_to_remote("${workdir}/${tess_id}/${skycell_id}"); + # This actually returns all the individual stack/skyfiles that comprise this run. Because consistency. + my $command = "$stacktool -inputskyfile -stack_id $stack_id "; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run stacktool -inputskyfile ", + $stack_id,$error_code); + } + + my $warpData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine stack component information.", + $stack_id,$PS_EXIT_PROG_ERROR); + my $warpData2= parse_md_list($warpData); + +# +# Step 3: Iterate over the sub-components + my $warp_path_base_string = ""; + foreach my $warpEntry ( @{ $warpData2 } ) { + my $warp_path_base = $warpEntry->{path_base}; + my $remote_path_base = uri_local_to_remote($warp_path_base); + $warp_path_base_string .= " $remote_path_base "; + + # Append file names to transfer lists so we can recover if warps are deleted. + uri_to_outputs($warp_path_base . ".fits"); + uri_to_outputs($warp_path_base . ".wt.fits"); + uri_to_outputs($warp_path_base . ".mask.fits"); + uri_to_outputs($warp_path_base . ".cmf"); +# uri_to_outputs($warp_path_base . ".psf"); + } + my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl $warp_path_base_string > ${remote_outroot}.in.mdc"; + my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use + my $ppstack_command = " ppStack -input ${remote_outroot}.in.mdc "; + $ppstack_command .= " ${remote_outroot} -stats ${remote_outroot}.stats "; + $ppstack_command .= " -recipe PPSTACK $recipe_ppstack "; + $ppstack_command .= " -recipe PPSUB STACK_1DG -recipe PSPHOT STACK -recipe PPSTATS STACKSTATS "; + $ppstack_command .= " -stack-type DEEP_STACK -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE "; + $ppstack_command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF "; + $ppstack_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL "; + $ppstack_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID -R PPSTACK.OUTPUT FITS.TYPE COMP_STACK "; + $ppstack_command .= " -R PPSTACK.OUTPUT.VARIANCE FITS.TYPE COMP_STACK -R PPSTACK.UNCONV FITS.TYPE COMP_STACK "; + $ppstack_command .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE COMP_STACK"; + $ppstack_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; + $ppstack_command .= " -threads 4 "; + $ppstack_command .= " -dumpconfig ${remote_outroot}.mdc "; + $ppstack_command .= " -stack_id $stack_id -skycell_id $skycell_id -tess_id $tess_id "; + + my $post_cmd_echo = " echo -n \"stacktool -addsumskyfile -stack_id $stack_id "; + $post_cmd_echo .= " -path_base $ipp_outroot -uri UNKNOWN "; + $post_cmd_echo .= " -dbname $dbname " if defined $dbname; + $post_cmd_echo .= " -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; + + my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - STACK_SKYCELL >> ${remote_outroot}.dbinfo "; + + print CONFIG "${mk_mdc_command} && ${ppstack_command} && ${post_cmd_echo} && ${post_cmd_SfM} "; + $job_index++; + + # Determine which output files need to be returned + foreach my $component(@return_component_list) { + my $filename = $ipprc->filename($component,$ipp_outroot); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} "; + } + +# my $cleanup_cmd = " (rm -f /scratch3/watersc1/sys_temp/${tess_id}.${skycell_id}.stk.${stack_id}.*conv.*.fits || true) "; +# print CONFIG " && ${cleanup_cmd} \n"; + + print CONFIG "\n"; +} +close(CONFIG); +close(TRANSFER); +close(CHECK); +close(RETURN); +close(GENERATE); + +# +# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. +my $proc_need = $job_index * $threads; # how many total processors do we need? +my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? +my $time_need = $job_index * $job_cost; # How many seconds will this take? + +my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill +my ($time_req,$node_req); +if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { + $time_req = $min_time; + $node_req = $min_nodes; +} +elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { + $time_req = $max_time; + $node_req = $max_nodes; + print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; +} +else { + $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; + $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; +} + +$time_req += 2; +$node_req *= 3; +open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR); +print COMMAND "#!/bin/tcsh\n"; +print COMMAND "##### Moab controll lines\n"; +print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES +print COMMAND "#MSUB -j oe\n"; +print COMMAND "#MSUB -V\n"; +print COMMAND "#MSUB -o ${remote_root}stack.${remote_id}.out\n"; +print COMMAND "date\n"; +print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; +print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 3' . "\n"; +print COMMAND "date\n"; +close(COMMAND); + +unless($no_update) { + my $command = "remotetool -updaterun -remote_id $remote_id "; + $command .= " -set_state pending "; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); +} + + +## Common SC routines + +sub uri_convert { # (ipp_disk,remote_disk) = uri_convert(neb_uri); + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_convert_and_create { # (ipp_disk,remote_disk) = uri_convert_and_create(neb_uri); ipp_disk is created if it doesn't exist + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_local_to_remote { #(remote_uri) = uri_local_to_remote(local_neb_uri); + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); + # This needs to replace the remote root directory with the nebulous tag. + my $remote_uri = shift; + $remote_uri =~ s%${remote_root}%%; + my $local_uri = "neb:///" . $remote_uri; + + return($local_uri); +} + + +sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status); + my $msg = shift; + my $id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $id and not $no_update) { + my $command = "remotetool -updaterun -remote_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_warp.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_warp.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_prepare_warp.pl (revision 37068) @@ -0,0 +1,378 @@ +#! /usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use DateTime; +use Data::Dumper; +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Hard coded values +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $threads = 2; # How many threads are we going to use? +my $job_cost = 70 / 60 / 60; # Estimate of how long a job runs, in hours. +my $proc_per_node = 24; # processors per node +my $min_nodes = 1; # smallest allocation to ask for +my $max_nodes = 1000; # largest allocation to ask for +my $min_time = 1; # shortest allocation to ask for +my $max_time = 8; # longest allocation to ask for +my $remote_hostname= "LANL/Mustang"; # Name of the remote node. +my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SAVE","LOG.EXP","TRACE.EXP"); +# Look for programs we need +my $missing_tools; + +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($remote_id,$warp_id,,$camera,$dbname,$verbose,$path_base,$no_update); +GetOptions( + 'remote_id=s' => \$remote_id, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, + 'path_base=s' => \$path_base, + 'no_update' => \$no_update, + 'verbose' => \$verbose, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless + defined($remote_id) and + defined($camera) and + defined($path_base) and + defined($dbname); +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# +# Step 1: Get a list of the components that make up this remoteRun + +# SHould this call listrun to ensure we're in state new? +my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; +$rt_cmd .= " -dbname $dbname " if defined($dbname); + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $rt_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run remotetool to determine stage parameters.", + $remote_id,$error_code); +} +my $compData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine component information.", + $remote_id,$PS_EXIT_PROG_ERROR); +my $compData2= parse_md_list($compData); + +# +# Step 1b: Open output files +my $uri_command = $path_base . ".cmd"; +my $uri_transfer= $path_base . ".transfer"; +my $uri_check = $path_base . ".check"; +my $uri_config = $path_base . ".config"; +my $uri_generate= $path_base . ".generate"; +my $uri_return = $path_base . ".return"; + +my $disk_command = $ipprc->file_resolve($uri_command,1); +my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); +my $disk_check = $ipprc->file_resolve($uri_check,1); +my $disk_config = $ipprc->file_resolve($uri_config,1); +my $disk_generate= $ipprc->file_resolve($uri_generate,1); +my $disk_return = $ipprc->file_resolve($uri_return,1); + +my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id,$PS_EXIT_SYS_ERROR); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$remote_id,$PS_EXIT_SYS_ERROR); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$remote_id,$PS_EXIT_SYS_ERROR); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id,$PS_EXIT_SYS_ERROR); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $PS_EXIT_SYS_ERROR); + + +# +# Step 2: Iterate over all componenets in this remote run. +my $job_index = 0; +my @pre_commands = (); +my @main_commands = (); +my @post_commands = (); + +my %file_filter = (); + +foreach my $compEntry (@$compData2) { + my $warp_id = $compEntry->{stage_id}; + +# Get exposure level information from the warpRun we're working from. + # This actually returns all the individual warp/skyfiles that comprise this run. Because consistency. + my $command = "$warptool -towarped -warp_id $warp_id "; + $command .= " -dbname $dbname " if defined($dbname); + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run warptool -pendingimfile ", + $warp_id,$error_code); + } + my $warpData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine warp component information.", + $warp_id,$PS_EXIT_PROG_ERROR); + my $warpData2= parse_md_list($warpData); + +# +# Step 3: Iterate over the sub-components + foreach my $warpEntry ( @{ $warpData2 } ) { + my $workdir = $warpEntry->{workdir}; + my $exp_tag = $warpEntry->{exp_tag}; + my $skycell_id = $warpEntry->{skycell_id}; + my $tess_id = $warpEntry->{tess_id}; + my $reduction = 'DEFAULT' unless defined $warpEntry->{reduction}; + my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use + my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.wrp.${warp_id}.${skycell_id}"; + print "$ipp_outroot\n"; + my $remote_outroot = uri_local_to_remote($ipp_outroot); + + my $reg_command = "$warptool -scmap -warp_id $warp_id -skycell_id ${skycell_id}"; + $reg_command .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $reg_command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run warptool -pendingimfile: $reg_command ", + $warp_id,$error_code); + } + + my $inpData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine warp component information.", + $warp_id, $PS_EXIT_PROG_ERROR); + my $inpData2=parse_md_list($inpData); + + my $pre_cmd_ims = "ls -1 "; + my $pre_cmd_masks= "ls -1 "; + my $pre_cmd_vars = "ls -1 "; + my $pre_cmd_astrom = ""; + + foreach my $imfile (@{ $inpData2 }) { + my $image = $ipprc->filename("PPIMAGE.CHIP", $imfile->{chip_path_base}, $imfile->{class_id}); + my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); + my $var = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); + + my $astrom= $ipprc->filename("PSASTRO.OUTPUT", $imfile->{cam_path_base}); + + my $remote_image = uri_local_to_remote($image); + my $remote_mask = uri_local_to_remote($mask); + my $remote_var = uri_local_to_remote($var); + my $remote_astrom= uri_local_to_remote($astrom); + + $pre_cmd_ims .= " $remote_image "; + $pre_cmd_masks .= " $remote_mask "; + $pre_cmd_vars .= " $remote_var "; + $pre_cmd_astrom = "echo $remote_astrom > ${remote_outroot}.astrom "; + } + $pre_cmd_ims .= " > ${remote_outroot}.imlist "; + $pre_cmd_masks .= " > ${remote_outroot}.masklist "; + $pre_cmd_vars .= " > ${remote_outroot}.varlist "; + + my $skycell_command = " dvoImageExtract -D CATDIR /turquoise/usr/projects/ps1/watersc1/tess/${tess_id} $skycell_id -o ${remote_outroot}.skyfile "; + + my $pswarp_command = " pswarp -list ${remote_outroot}.imlist "; + $pswarp_command .= " -masklist ${remote_outroot}.masklist -variancelist ${remote_outroot}.varlist "; + $pswarp_command .= " -astromlist ${remote_outroot}.astrom "; + $pswarp_command .= " ${remote_outroot} ${remote_outroot}.skyfile "; + $pswarp_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE "; + $pswarp_command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF "; + $pswarp_command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF "; + $pswarp_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS "; + $pswarp_command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL "; + $pswarp_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID "; + $pswarp_command .= " -recipe PSWARP $recipe_pswarp "; + $pswarp_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; + $pswarp_command .= " -threads 4 "; # -image_id ${image_id} -source_id ${source_id} "; + $pswarp_command .= " -recipe PPSTATS WARPSTATS "; + $pswarp_command .= " -dumpconfig ${remote_outroot}.mdc -stats ${remote_outroot}.stats "; + +# print "$pswarp_command \n"; + + my $post_cmd_echo = " echo -n \"warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id "; + $post_cmd_echo .= " -path_base $remote_outroot -uri UNKNOWN "; + $post_cmd_echo .= " -dbname $dbname " if defined $dbname; + $post_cmd_echo .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; + + my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - WARP_SKYCELL >> ${remote_outroot}.dbinfo "; + + print CONFIG "${pre_cmd_ims} && ${pre_cmd_masks} && ${pre_cmd_vars} && ${pre_cmd_astrom} && ${skycell_command} && ${pswarp_command} && ${post_cmd_echo} && ${post_cmd_SfM} "; + $job_index++; + + # Determine which output files need to be returned + foreach my $component(@return_component_list) { + my $filename = $ipprc->filename($component,$ipp_outroot); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} "; + } + + print CONFIG "\n"; +# die(); + } +} +close(CONFIG); +close(TRANSFER); +close(CHECK); +close(RETURN); +close(GENERATE); + +# +# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. +my $proc_need = $job_index * $threads; # how many total processors do we need? +my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? +my $time_need = $job_index * $job_cost; # How many seconds will this take? + +my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill +my ($time_req,$node_req); +if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { + $time_req = $min_time; + $node_req = $min_nodes; +} +elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { + $time_req = $max_time; + $node_req = $max_nodes; + print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; +} +else { + $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; + $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; +} + +open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR); +print COMMAND "#!/bin/tcsh\n"; +print COMMAND "##### Moab controll lines\n"; +print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES +print COMMAND "#MSUB -j oe\n"; +print COMMAND "#MSUB -V\n"; +print COMMAND "#MSUB -o ${remote_root}warp.${remote_id}.out\n"; +print COMMAND "date\n"; +print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; +print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; +print COMMAND "date\n"; +close(COMMAND); + +unless($no_update) { + my $command = "remotetool -updaterun -remote_id $remote_id "; + $command .= " -set_state pending "; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); +} + + +## Common SC routines + +sub uri_convert { # (ipp_disk,remote_disk) = uri_convert(neb_uri); + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_convert_and_create { # (ipp_disk,remote_disk) = uri_convert_and_create(neb_uri); ipp_disk is created if it doesn't exist + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_local_to_remote { #(remote_uri) = uri_local_to_remote(local_neb_uri); + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); + # This needs to replace the remote root directory with the nebulous tag. + my $remote_uri = shift; + $remote_uri =~ s%${remote_root}%%; + my $local_uri = "neb:///" . $remote_uri; + + return($local_uri); +} + + +sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status); + my $msg = shift; + my $id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $id and not $no_update) { + my $command = "remotetool -updaterun -remote_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_remote_exec.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_remote_exec.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_remote_exec.pl (revision 37068) @@ -0,0 +1,398 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use File::Basename; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Hard coded values +my $DMZ_HOST = 'wtrw'; +my $SEC_HOST = 'mu-fe4'; +my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/'; +my $remote_root = '/scratch3/watersc1/'; + +# tools +my $missing_tools; +my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); +my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Options +my ($remote_id,$path_base,$policy,$poll,$job_id,$dbname,$verbose,$no_update,$camera); + +GetOptions( + 'remote_id=s' => \$remote_id, + 'job_id=s' => \$job_id, + 'path_base=s' => \$path_base, + 'policy=s' => \$policy, + 'poll' => \$poll, + 'camera=s' => \$camera, + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'no_update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --path_base", -exitval => 3) unless + defined($path_base) and + defined($remote_id); + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Phase 1: See if we can actually do anything. +# If the link is down, there's no benefit in trying to do anything else. +&check_ssh_connection(); + +print "passed authentication challenge.\n"; + +# These are operations we only need to do on the first call, not on subsequent poll operations. +unless ($poll) { +# Phase 2: Ensure files are in place: +# Copy command files + my @files = (); + + my $uri_command = $path_base . ".cmd"; + my $uri_transfer= $path_base . ".transfer"; + my $uri_check = $path_base . ".check"; + my $uri_config = $path_base . ".config"; + my $uri_generate= $path_base . ".generate"; + my $uri_return = $path_base . ".return"; + + my $disk_command = $ipprc->file_resolve($uri_command); + my $disk_transfer= $ipprc->file_resolve($uri_transfer); + my $disk_check = $ipprc->file_resolve($uri_check); + my $disk_config = $ipprc->file_resolve($uri_config); + my $disk_generate= $ipprc->file_resolve($uri_generate); + my $disk_return = $ipprc->file_resolve($uri_return); + + scp_put($disk_command,uri_local_to_remote($uri_command)); + scp_put($disk_transfer,uri_local_to_remote($uri_transfer)); + scp_put($disk_check,uri_local_to_remote($uri_check)); + scp_put($disk_config,uri_local_to_remote($uri_config)); + scp_put($disk_generate,uri_local_to_remote($uri_generate)); + scp_put($disk_return, uri_local_to_remote($uri_return)); + +# Run check command + my (undef,$remote_check) = uri_convert($uri_check); + my (undef,$remote_transfer)= uri_convert($uri_transfer); +# my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_validate_files.pl --check_list $remote_check --transfer_list $remote_transfer"); + my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer"); + # We no longer need to parse this output, as it retrieves files itself. + foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) { + print "$l\n"; + } + +# Run real command + my (undef,$remote_command) = uri_convert($uri_command); + my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command"); + if ($#{ $ssh_exec_stdout } != -1) { # Parse the output + $job_id = ${ $ssh_exec_stdout }[0]; + chomp($job_id); + $job_id =~ s/\s+//g; + } + + unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. + &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); + } + + # Notify the database that this entry is currently running. + my $command = "$remotetool -updaterun -remote_id $remote_id "; + $command .= " -set_state run "; + $command .= " -job_id $job_id "; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + +} # End of exec phase + +# Check that we have a valid job_id, either from the command line, or from the msub command. +unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. + &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); +} + + +# Poll the job status? +my $poll_count = 0; +my $poll_max = 5; +my $poll_sleep = 60; +my $poll_response = 0; +while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)) { + if ($verbose) { + print "Polling code. Sleeping. $poll_count of $poll_max $poll_response\n"; + } + unless ($poll_count == 0) { + sleep($poll_sleep); + } + $poll_response = poll_job($job_id); + + if ($poll_response == 1) { # This job has completed + last; + } + elsif ($poll_response == -1) { # This job has an error + &my_die("The job exited incorrectly.",$remote_id, $PS_EXIT_PROG_ERROR); + } + $poll_count++; +} +if ($verbose) { + print "Stopped Polling code. $poll_count of $poll_max $poll_response\n"; +} +if ($poll_response != 1) { + &my_die("Job hasn't completed. Ending polling.", $remote_id, 0, "run", $job_id); +} +# Retrieve validation summary +my $fault = 0; + +# Initialize the remote side to sync things + +# Once sync completes, execute all the database commands +# Grab the return list +my $uri_return = $path_base . ".return"; +my ($ipp_return,$remote_return) = uri_convert($uri_return); +scp_get($remote_return,$ipp_return); +# Feed teh return list into the +my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return"); +print "$return_push_output\n"; + + +open(RETURN, "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id,$PS_EXIT_SYS_ERROR); +while() { + chomp; + if ($_ =~ /dbinfo/) { + my $file = $_; +# $file =~ s/${remote_root}.tmp//; THis shouldn't be needed, as the transfer tool munges the files on that side. + open(DBF,"$file") || warn "Missing file $file\n"; + my $cmd = ; + chomp($cmd); + $cmd =~ s/-/ -/g; # This is just a safety check for missing space. + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + # This shouldn't fail, but we also can't suddenly abort if something does fail. Maybe retry if this looks to be a continuing issue? + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("The command that shouldn't fail has failed: $cmd $remote_id $error_code");##,$remote_id,$error_code); + } + close(DBF); + } +} +close(RETURN); + +# We're done, so set the state and exit. +&my_die("Finished",$remote_id,0,"full"); + + + +# END PROGRAM + + +sub check_ssh_connection { + my $cmd = "$ssh -O check $DMZ_HOST"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Authorization check failed.",$remote_id,0,'auth'); + +# my $update_command = "$remotetool -updaterun -remote_id $remote_id "; +# $update_command .= " -set_state auth "; +# $update_command .= " -dbname $dbname " if defined $dbname; +# system($update_command); +# exit(0); # This isn't an error. It's a state we monitor and move to automatically. + } +} + +sub scp_put { + my $file = shift; + my $destination = shift; + my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}"; + + my $directory = dirname($destination); + ssh_exec_command("mkdir -p $directory"); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die(); + } +} + +sub scp_get { + my $destination = shift; + my $file = shift; + + my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file "; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die(); + } +} + +sub ssh_exec_command { + my $cmd = shift; + $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Failed to execute command: >>$cmd<<"); + } + return ($stdout_buf); +} + +sub poll_job { + my $job_id = shift; + my $response_array = ssh_exec_command("checkjob -v $job_id"); + my $response = join "\n", @$response_array; + + if ($response =~ /State: Running/) { + return(0); + } + elsif ($response =~ /State: Completed/) { + return(1); + } + elsif ($response =~ /State: Idle/) { + return(-2); + } + else { + return(-1); + } +} + + +sub uri_convert { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + +# print TRANSFER "$ipp_disk\n"; +# print CHECK "$remote_disk\n"; + return($ipp_disk,$remote_disk); +} + +sub uri_local_to_remote { + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub uri_remote_to_local { + # This needs to replace the remote root directory with the nebulous tag. + my $remote_uri = shift; + $remote_uri =~ s%${remote_root}%%; + my $local_uri = "neb:///" . $remote_uri; + + return($local_uri); +} +sub my_die { + my $msg = shift; + my $id = shift; + my $exit_code = shift; + my $exit_state = shift; + my $jobid = shift; + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $id and not $no_update) { + my $command = "$remotetool -updaterun -remote_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -job_id $job_id " if defined $jobid; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} + +# I don't think I need this anymore, but I don't want to burn it until I"m sure. + +# exit(1); +# # Read this as a metadata object +# my $ssh_exec_stdout = ssh_exec_command("sc_validate_processing.pl ${path_base}.out"); +# my $retrieveData = $mdcParser->parse(join "", @$ssh_exec_stdout); +# my $retrieveData2= parse_md_list($retrieveData); + +# foreach my $retrieveEntry (@$retrieveData2) { +# my $process_fault = $retrieveEntry->{fault}; +# my $dbcommand = $retrieveEntry->{dbcommand}; +# my $filelist = $retrieveEntry->{filelist}; + +# scp_get("${filelist}",${filelist}); +# open(FL,${filelist}); +# while() { +# chomp; +# my $f = $_; +# scp_get("${f}",$f); +# } +# close(FL); + +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $dbcommand, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("The database command returned did not execute correctly. $dbcommand $process_fault", +# $remote_id,$error_code); +# } + +# if ($process_fault != 0) { +# $fault = 1; +# } +# } + +# if (($policy ne 'ignore')&&($fault != 0)) { +# &my_die("This job exited without all components finishing successfully. Halting.", +# $remote_id,$fault,"halt"); +# } +# else { +# print "This job exited without all components finishing successfully. $fault\n" if $fault; +# &my_die("This job finished. Setting remote job to full.", +# $remote_id,0,"full"); +# } + +# Promote to finished. +#&my_die(); # Not really, but I don't have the tool interface finished, so these are all just placeholders. +# Quick review: +# new -> pending -> run -> full +# auth Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_transfer_tool.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_transfer_tool.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_transfer_tool.pl (revision 37068) @@ -0,0 +1,172 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use File::Basename; +use IPC::Cmd 0.36 qw( can_run run ); +use Sys::Hostname; + +my $local_raw = '/scratch3/watersc1/raw/'; +my $local_tmp = '/scratch3/watersc1/tmp/'; +my $threads = 10; +my @hosts = ('ippc20.ipp.ifa.hawaii.edu','ippc24.ipp.ifa.hawaii.edu','ippc28.ipp.ifa.hawaii.edu', + 'ippc21.ipp.ifa.hawaii.edu','ippc25.ipp.ifa.hawaii.edu','ippc29.ipp.ifa.hawaii.edu', + 'ippc22.ipp.ifa.hawaii.edu','ippc26.ipp.ifa.hawaii.edu', + 'ippc23.ipp.ifa.hawaii.edu','ippc27.ipp.ifa.hawaii.edu' ); +@hosts = (@hosts, @hosts, @hosts, @hosts, @hosts); +my $input_file; +my $verbose = 0; +my $fetch = 0; +my $offset = 0; + +GetOptions( + 'threads=s' => \$threads, + 'input=s' => \$input_file, + 'fetch=s' => \$fetch, + 'offset=s' => \$offset, + 'verbose' => \$verbose, + ) or pod2usage( 2 ); +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV; +pod2usage( -msg => "Required options: --input", -exitval => 3) unless + defined($input_file); + +my $hostname = hostname; +print STDERR "Running on $hostname\n"; + +# split in input file list +unless(-d $local_tmp) { system("mkdir -p $local_tmp"); } +unless(-d $local_raw) { system("mkdir -p $local_raw"); } + +open(I,$input_file) || die "Couldn't find input file specified\n"; +if ($fetch) { + open(F,$fetch) || die "Couldn't find fetch file specified\n"; +} +my $input_base = basename($input_file); +my @filehandles; +my $i; +my $line = 0; +for ($i = 0; $i < $threads; $i++) { + open($filehandles[$i], ">${local_tmp}/${input_base}.${i}"); +} +$i = 0; +while() { + chomp; + $line++; + if ($line < $offset) { next; } # I think this is off-by-one in a safe direction + + $i = int(rand($#filehandles + 1)); + + my $fline; + if ($fetch) { + $fline = ; + } + if (($fetch)&&(!(-e $_))) { # We are fetching, and do not already have this file. + print { $filehandles[$i] } $fline; + } + elsif (!($fetch)) { # We are pushing + print { $filehandles[$i] } "${local_tmp}/$_" . "\n"; + } +} +close(I); + +for ($i = 0; $i < $threads; $i++) { + close($filehandles[$i]); +} + +# fork the tars +my @pids = (); +for ($i = 0; $i < $threads; $i++) { + $pids[$i] = fork(); + if ($pids[$i] == 0) { + my $host = $hosts[$i]; + my $code = 0; + if ($fetch) { + $code = fetch_task($host,"${input_base}.${i}", 0); + } + else { + $code = transfer_task($host,"${local_tmp}/${input_base}.${i}", 0); + } + exit($code); + } +} +for ($i = 0; $i < $threads; $i++) { + waitpid($pids[$i],0); +} + +for ($i = 0; $i < $threads; $i++) { +# unlink("${local_tmp}/${input_base}.${i}"); +} + + +# distribute bundles to nodes + +# execute bundle transfer + +sub transfer_task { + my $destination_host = shift; + my $transfer_filelist= shift; + my $error = shift; + + # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date? + my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; + print STDERR "$command\n"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + + print STDERR "ERROR: $error_code\n"; +# my $std = join "\n", @{ $stdout_buf }; +# print "STDOUT: $std\n"; +# $std = join "\n", @{ $stderr_buf }; +# print "STDERR: $std\n"; + + foreach my $line (@{ $stderr_buf }) { # This is a hack. A messy ugly hack. + if ($line =~ /No such file or directory/) { + my $file = (split /\s+/, $line)[1]; + $file =~ s/:$//; + if ($file !~ /nebulous/) { next; } + print STDERR "Touching $file so tar can continue.\n"; + system("touch $file"); + } + } + + warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code."); + $error++; + if ($error < 4) { + $error_code = transfer_task($destination_host,$transfer_filelist,$error); + } + else { + die("Failed too many times $error $destination_host $transfer_filelist"); + } + } + return($error_code); +} + +sub fetch_task { + my $destination_host = shift; + my $transfer_filelist= shift; + my $error = shift; + + system("/usr/projects/cosmo/amd6100/bin/scp -o NoneSwitch=yes -o NoneEnabled=yes ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/"); + my $command = "/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar cf - --ignore-failed-read --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference "; + print STDERR "$command\n"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or 4); + warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code."); + $error++; + if ($error < 4) { + $error_code = fetch_task($destination_host,$transfer_filelist,$error); + } + else { + die("Failed too many times $error $destination_host $transfer_filelist"); + } + } + return($error_code); +} Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_validate_files.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_validate_files.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_validate_files.pl (revision 37068) @@ -0,0 +1,68 @@ +#! /usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use File::Basename; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Hard coded values +my $remote_root = '/scratch3/watersc1/'; +my $host_iter = 0; +my @ipp_hosts = ('166.122.172.41','166.122.172.42','166.122.172.43','166.122.172.44', + '166.122.172.45','166.122.172.46','166.122.172.47','166.122.172.48', + '166.122.172.49','166.122.172.50','166.122.172.51','166.122.172.52','166.122.172.53'); +my @remote_hosts = ('mu-fe1.lanl.gov','mu-fe2.lanl.gov','mu-fe3.lanl.gov','mu-fe4.lanl.gov'); +my $check_list; +my $transfer_list; +GetOptions( + 'check_list=s' => \$check_list, + 'transfer_list=s' => \$transfer_list + ) or pod2usage( 2 ); + +my @check_files = (); +my @transfer_files = (); + +open(LIST,$check_list); +while() { + chomp; + push @check_files, $_; +} +close(LIST); +open(LIST2,$transfer_list); +while() { + chomp; + push @transfer_files, $_; +} +close(LIST2); + +unless ($#check_files == $#transfer_files) { + die "The two lists are of unequal length."; +} + +for (my $i = 0; $i <= $#check_files; $i++) { + my $file = $check_files[$i]; + unless (-e $file) { + print "$file\n"; + my $fetch_file = $transfer_files[$i]; + my $directory = dirname($file); + unless (-d $directory) { + system("mkdir -p $directory"); + } + + scp_pull($fetch_file,$file); + } +} + +sub scp_pull { + my $remote_file = shift; + my $local_file = shift; + + my $host = $ipp_hosts[$host_iter]; + $host_iter++; + if ($host_iter > $#ipp_hosts) { $host_iter = 0; } + system("scp ${host}:${remote_file} ${local_file}"); +} Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_validate_processing.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_validate_processing.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/sc_validate_processing.pl (revision 37068) @@ -0,0 +1,79 @@ +#! /usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Hard coded values +my $remote_root = '/scratch3/watersc1/'; + +my $command_output +GetOptions( + 'command_output=s' => \$command_output; + ) or pod2usage( 2 ); + +open(LIST,$command_output); +while() { + chomp; + + if ($_ =~ /ppImage/) { + my $base = (split /\s+/)[15]; # This is super fragile. + my $stats= $base . "." . $class_id . ".stats"; # no good idea how to get that. + my $command = ""; + if (-e $stats) { + my $stat_cmd = "ppStatsFromMetadata $stats - CHIP_IMFILE"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $exp_id, $chip_id, $class_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + + # Need to get all these parameters. + $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -chip_id $chip_id"; + $command .= " -class_id $class_id"; + $command .= " -uri $outputImage"; + $command .= " -path_base $outroot"; + $command .= " -magicked $magicked" if $magicked; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + + } + else { # This job failed permanently + $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -uri $outputImage" if defined $outputImage; + $command .= " -path_base $outroot"; + $command .= " -hostname $host" if defined $host; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -chip_id $chip_id"; + $command .= " -class_id $class_id"; + $command .= " -fault $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + } + + +# if ($_ =~ /stask: error: .*: task .*: Exited with exit code/) { +# print " "; # This needs to be smarter. +# exit(1); +# } +} +close(LIST); + Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/stack_bkg_mk_mdc.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/stack_bkg_mk_mdc.pl (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/stack_bkg_mk_mdc.pl (revision 37068) @@ -0,0 +1,91 @@ +#! /usr/bin/env perl + +use warnings; +use strict; +use Carp; +use DBI; +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +my ($dbname,$stack_id,$label,$camera,$filter); +#$dbname = 'gpc1'; +#$camera = 'GPC1'; +GetOptions( + 'stack_id=s' => \$stack_id, + 'label=s' => \$label, + 'filter=s' => \$filter, + 'dbname=s' => \$dbname, + 'camera=s' => \$camera, + ) or pod2usage ( 2 ); +pod2usage( -msg => "Required options; --stack_id | --label", -exitval => 3) unless + ((defined $stack_id)||(defined $label)); +pod2usage( -msg => "Required options; --camera --dbname ", -exitval => 3) unless + (defined($camera) && defined($dbname)); +my $verbose = 0; + +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; +my $dbserver = 'ippdb01'; +my $dbuser = 'ippuser'; +my $dbpass = 'ippuser'; +my $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + +my $ipprc = PS::IPP::Config->new( $camera ); +my $mdcParser = PS::IPP::Metadata::Config->new; + + +my $query = "SELECT DISTINCT chip_id,cam_id,path_base FROM stackRun JOIN stackInputSkyfile USING(stack_id) JOIN warpRun USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun USING (chip_id) WHERE 1 "; + +if (defined($label)) { $query .= " AND stackRun.label = '${label}' "; } +if (defined($stack_id)) { $query .= " AND stack_id = ${stack_id} "; } +if (defined($filter)) { $query .= " AND stackRun.filter = '${filter}' "; } + +my $cams = $db->selectall_arrayref( $query ); + +#my $logDest = $ipprc->filename($logRule, $outroot, $class_id); + +print "backgroundStackInputs MULTI\n\n"; +my $index = 0; + +foreach my $cam (@{ $cams }) { + my ($chip_id,$cam_id,$cam_path) = @{ $cam }; + my $smf_file = $ipprc->filename("PSASTRO.OUTPUT",$cam_path); + print "backgroundStackInput${index} METADATA\n"; + print " chip_id S64 $chip_id\n"; + print " cam_id S64 $cam_id\n"; + print " astrom STR $smf_file\n"; + print " models METADATA\n"; + + my $command = "chiptool -processedimfile -chip_id $chip_id"; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die "Unable to parse metadata."; + } + my $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + die "Unabel to parse metadata: 2."; + + foreach my $imf (@$imfiles) { + my $class_id = $imf->{class_id}; + my $path_base= $imf->{path_base}; + + my $model = $ipprc->filename("PPIMAGE.BACKMDL",$path_base,$class_id); + + print " ${class_id} STR $model\n"; + } + print " END\n"; + print "END\n\n"; + + $index++; +} + Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/stack_skycell.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/stack_skycell.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/stack_skycell.pl (revision 37068) @@ -40,5 +40,5 @@ } -my ($stack_id, $dbname, $outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); +my ($stack_id, $dbname, $outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps, $delete_convolved_images); GetOptions( 'stack_id|d=s' => \$stack_id, # Stack identifier @@ -55,4 +55,5 @@ 'redirect-output' => \$redirect, 'save-temps' => \$save_temps, # Save temporary files? + 'delete-convolved' => \$delete_convolved_images, ) or pod2usage( 2 ); @@ -92,4 +93,5 @@ ); + my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $stack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration $| = 1; @@ -360,5 +362,25 @@ if (!$quality) { + + check_outputs(\@outputFiles, $replicate_outputs); + + if (($convolve)&&($delete_convolved_images)) { # We made convolved products, but do not wish to keep them anymore + my @products_to_clear = ('PPSTACK.OUTPUT', + 'PPSTACK.OUTPUT.MASK', + 'PPSTACK.OUTPUT.VARIANCE', + 'PPSTACK.OUTPUT.EXP', + 'PPSTACK.OUTPUT.EXPNUM', + 'PPSTACK.OUTPUT.EXPWT'); + foreach my $product (@products_to_clear) { + my $file = $ipprc->filename($product,$outroot,$skycell_id); + print "Deleting unwanted convolved product: $file\n"; + my $error= $ipprc->kill_file($file); + if ($error_code) { + print "Failed to delete unwanted convolved product: $error_code\n"; + } + } + } + } Index: branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/staticsky.pl =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/staticsky.pl (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/staticsky.pl (revision 37068) @@ -138,4 +138,6 @@ my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot); + my $needConvolvedImages = 0; + foreach my $file (@$files) { print $listFile "INPUT METADATA\n"; @@ -146,8 +148,11 @@ my $stack_id = $file->{stack_id}; - my $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name - my $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base ); # Mask name - my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name - my $expnumCnv = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM", $path_base ); # Expnum name + my ($imageCnv, $maskCnv, $weightCnv, $expnumCnv); + if ($needConvolvedImages) { + $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name + $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base ); # Mask name + $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name + $expnumCnv = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM", $path_base ); # Expnum name + } my $imageRaw = $ipprc->filename("PPSTACK.UNCONV", $path_base ); # Image name @@ -166,9 +171,11 @@ &my_die("Couldn't find input: $weightRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw"); &my_die("Couldn't find input: $expnumRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumRaw"); - &my_die("Couldn't find input: $imageCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv"); - &my_die("Couldn't find input: $maskCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv"); - &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv"); - &my_die("Couldn't find input: $expnumCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumCnv"); - &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); + if ($needConvolvedImages) { + &my_die("Couldn't find input: $imageCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv"); + &my_die("Couldn't find input: $maskCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv"); + &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv"); + &my_die("Couldn't find input: $expnumCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumCnv"); + &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); + } &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources"); @@ -178,12 +185,12 @@ print $listFile " RAW:VARIANCE STR " . $weightRaw . "\n"; print $listFile " RAW:EXPNUM STR " . $expnumRaw . "\n"; - - print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; - print $listFile " CNV:MASK STR " . $maskCnv . "\n"; - print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; - print $listFile " CNV:EXPNUM STR " . $expnumCnv . "\n"; - print $listFile " CNV:PSF STR " . $psfCnv . "\n"; - print $listFile " SOURCES STR " . $sources . "\n"; + if ($needConvolvedImages) { + print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; + print $listFile " CNV:MASK STR " . $maskCnv . "\n"; + print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; + print $listFile " CNV:EXPNUM STR " . $expnumCnv . "\n"; + print $listFile " CNV:PSF STR " . $psfCnv . "\n"; + } print $listFile "END\n\n";