Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_camera.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_camera.pl (revision 37321) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_camera.pl (revision 37322) @@ -74,5 +74,5 @@ my $disk_return = $ipprc->file_resolve($uri_return,1); -my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. +my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it. open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); @@ -139,5 +139,5 @@ my $chipProto = $metadata->[0]; my $chip_path = $chipProto->{path_base}; - my $remote_chip_path = uri_local_to_remote( $chip_path ); + my $remote_chip_path = &uri_local_to_remote( $chip_path ); my $pre_cmd_cmfs = "ls -1 ${remote_chip_path}*.cmf > ${remote_outroot}.cmflist"; my $pre_cmd_masks= "ls -1 ${remote_chip_path}*.mk.fits > ${remote_outroot}.masklist"; @@ -172,5 +172,5 @@ foreach my $component(@return_component_list) { my $filename = $ipprc->filename($component,$ipp_outroot); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename); my $remote_outroot_dir = dirname($ipp_disk); print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; @@ -178,5 +178,5 @@ foreach my $chipInfo (@{ $metadata }) { my $filename = $ipprc->filename("PSASTRO.OUTPUT.MASK",$ipp_outroot,$chipInfo->{class_id}); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename); my $remote_outroot_dir = dirname($ipp_disk); print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl (revision 37321) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl (revision 37322) @@ -78,5 +78,5 @@ my $disk_return = $ipprc->file_resolve($uri_return,1); -my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. +my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it. open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); @@ -93,6 +93,6 @@ my $command = "$stacktool -tosum -stack_id $stack_id"; $command .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -113,6 +113,6 @@ } my $ipp_outroot = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.stk.${stack_id}"; -my $remote_outroot = uri_local_to_remote($ipp_outroot); -my $remote_outdir = uri_local_to_remote("${workdir}/${tess_id}/${skycell_id}"); +my $remote_outroot = &uri_local_to_remote($ipp_outroot); +my $remote_outdir = &uri_local_to_remote("${workdir}/${tess_id}/${skycell_id}"); # STEP 2: Get the list of input warps @@ -122,6 +122,6 @@ my $command = "$stacktool -inputskyfile -stack_id $stack_id "; $command .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -131,5 +131,5 @@ my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state); - my $warpData = parse_md_list($MDlist); + $warpData = parse_md_list($MDlist); } @@ -139,13 +139,15 @@ foreach my $warpEntry ( @{ $warpData } ) { my $warp_path_base = $warpEntry->{path_base}; - my $remote_path_base = uri_local_to_remote($warp_path_base); + my $remote_path_base = &uri_local_to_remote($warp_path_base); $warp_path_base_string .= " $remote_path_base "; + if ($verbose) { print STDERR "$warp_path_base_string\n"; } + # Append file names to transfer lists so we can recover if warps are deleted. - uri_to_outputs($warp_path_base . ".fits"); - uri_to_outputs($warp_path_base . ".wt.fits"); - uri_to_outputs($warp_path_base . ".mask.fits"); - uri_to_outputs($warp_path_base . ".cmf"); -# uri_to_outputs($warp_path_base . ".psf"); + &uri_to_outputs($warp_path_base . ".fits"); + &uri_to_outputs($warp_path_base . ".wt.fits"); + &uri_to_outputs($warp_path_base . ".mask.fits"); + &uri_to_outputs($warp_path_base . ".cmf"); +# &uri_to_outputs($warp_path_base . ".psf"); } my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl $warp_path_base_string > ${remote_outroot}.in.mdc"; @@ -179,5 +181,5 @@ foreach my $component(@return_component_list) { my $filename = $ipprc->filename($component,$ipp_outroot); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename); my $remote_outroot_dir = dirname($ipp_disk); print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; @@ -203,4 +205,5 @@ ## Common SC routines +# KEEP sub uri_convert { # (ipp_disk,remote_disk) = uri_convert(neb_uri); my $neb_uri = shift; @@ -219,4 +222,5 @@ } +# KEEP sub uri_convert_and_create { # (ipp_disk,remote_disk) = uri_convert_and_create(neb_uri); ipp_disk is created if it doesn't exist my $neb_uri = shift; @@ -235,27 +239,25 @@ } +# KEEP sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print TRANSFER "$ipp_disk\n"; - print CHECK "$remote_disk\n"; - } - return($ipp_disk,$remote_disk); -} - + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + return($ipp_disk,$remote_disk); +} + +# KEEP sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print RETURN "$ipp_disk\n"; - print GENERATE "$remote_disk\n"; - } - return($ipp_disk,$remote_disk); -} - + + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; + return($ipp_disk,$remote_disk); +} + +# KEEP sub uri_local_to_remote { #(remote_uri) = uri_local_to_remote(local_neb_uri); # This needs to replace the nebulous tag with the remote root. @@ -269,14 +271,4 @@ } -sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); - # This needs to replace the remote root directory with the nebulous tag. - my $remote_uri = shift; - $remote_uri =~ s%${remote_root}%%; - my $local_uri = "neb:///" . $remote_uri; - - return($local_uri); -} - - sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status); my $msg = shift; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl (revision 37321) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl (revision 37322) @@ -74,5 +74,5 @@ my $disk_return = $ipprc->file_resolve($uri_return,1); -my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. +my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it. open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); @@ -90,10 +90,12 @@ my $command = "$warptool -towarped -warp_id $warp_id "; $command .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to run warptool -pendingimfile ", $remote_id, $warp_id, $error_code, $fail_state); } + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state); @@ -108,15 +110,22 @@ my $skycell_id = $warpEntry->{skycell_id}; my $tess_id = $warpEntry->{tess_id}; - my $reduction = 'DEFAULT' unless defined $warpEntry->{reduction}; + + my $reduction = $warpEntry->{reduction}; + $reduction = 'DEFAULT' unless defined $warpEntry->{reduction}; + my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use + if ($recipe_pswarp eq "") { + &my_die("Unable to determine PSWARP recipe for $reduction", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); + } + my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.wrp.${warp_id}.${skycell_id}"; print "$ipp_outroot\n"; - my $remote_outroot = uri_local_to_remote($ipp_outroot); + my $remote_outroot = &uri_local_to_remote($ipp_outroot); my $command = "$warptool -scmap -warp_id $warp_id -skycell_id ${skycell_id}"; $command .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -133,5 +142,5 @@ my $pre_cmd_astrom = ""; - foreach my $imfile (@{ $inData2 }) { + foreach my $imfile (@{ $inData }) { my $image = $ipprc->filename("PPIMAGE.CHIP", $imfile->{chip_path_base}, $imfile->{class_id}); my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); @@ -140,8 +149,8 @@ my $astrom= $ipprc->filename("PSASTRO.OUTPUT", $imfile->{cam_path_base}); - my $remote_image = uri_local_to_remote($image); - my $remote_mask = uri_local_to_remote($mask); - my $remote_var = uri_local_to_remote($var); - my $remote_astrom= uri_local_to_remote($astrom); + my $remote_image = &uri_local_to_remote($image); + my $remote_mask = &uri_local_to_remote($mask); + my $remote_var = &uri_local_to_remote($var); + my $remote_astrom= &uri_local_to_remote($astrom); $pre_cmd_ims .= " $remote_image "; @@ -187,5 +196,5 @@ foreach my $component(@return_component_list) { my $filename = $ipprc->filename($component,$ipp_outroot,$skycell_id); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename); my $remote_outroot_dir = dirname($ipp_disk); print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; @@ -248,11 +257,8 @@ sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles my $neb_uri = shift; - my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print TRANSFER "$ipp_disk\n"; - print CHECK "$remote_disk\n"; - } + my ($ipp_disk, $remote_disk) = &uri_convert( $neb_uri ); + + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; return($ipp_disk,$remote_disk); } @@ -260,10 +266,8 @@ sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE my $neb_uri = shift; - my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print RETURN "$ipp_disk\n"; - print GENERATE "$remote_disk\n"; - } + my ($ipp_disk, $remote_disk) = &uri_convert_and_create( $neb_uri ); + + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; return($ipp_disk,$remote_disk); } Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_poll.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_poll.pl (revision 37321) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_poll.pl (revision 37322) @@ -35,8 +35,10 @@ # tools my $missing_tools; -my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); -my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); +my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); +my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); -my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); if ($missing_tools) { @@ -167,4 +169,5 @@ my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($words[0]); + print STDERR "stage_id : $stage_id, class_id: $class_id\n"; if ($words[1] eq "PASS") { @@ -184,62 +187,98 @@ # STEP 7: if dbinfo is PASS, execute it (fail ones that are not PASS?). -foreach my $file (@dbinfo_files) { - open(DBF,"$file") || warn "Missing file $file\n"; - my $cmd = ; - close(DBF); - chomp($cmd); - - $cmd =~ s/-/ -/g; # This is just a safety check for missing space. - - if ($cmd eq "") { - print STDERR "problem: empty dbinfo file\n"; - exit (1); - next; - } - - print "dbinfo: $file\n"; - - if ($retry) { - # use the dbinfo filename to get stage id, check if we have already updated this entry: - my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); - - # we just need to check for the existence of an entry: - if ($runData->{stage} eq "chip") { - my $cmd1 = "$chiptool -simple -processedimfile -chip_id $stage_id -class_id $class_id"; - $cmd1 .= " -dbname $dbname " if defined($dbname); - - my ( $success1, $error_code1, $full_buf1, $stdout_buf1, $stderr_buf1 ) = - run(command => $cmd1, verbose => 0); - - unless ($success1) { - $error_code1 = (($error_code1 >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run chiptool to check on results", $remote_id, $error_code1); - } - - my $Nrows = @$stdout_buf1; - if ($Nrows > 0) { - print STDERR "already updated: $stage_id, $class_id\n"; - next; - } - } else { - print STDERR "do not know how to handle stage $runData->{stage}\n"; - exit (3); - } - } - - unless ($no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $cmd, verbose => $verbose); - unless ($success) { - # This shouldn't fail, but we also can't suddenly abort if something does fail. - # Now we're going to drop the component that owns this dbinfo, which should force a retry. - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); - my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); - $stage_fail{$stage_id} = 1; - } - } -} -print STDERR "dbinfo files all run (or skipped)\n"; +if (1) { + foreach my $file (@dbinfo_files) { + open(DBF,"$file") || warn "Missing file $file\n"; + my $cmd = ; + close(DBF); + chomp($cmd); + + $cmd =~ s/-/ -/g; # This is just a safety check for missing space. + + if ($cmd eq "") { + print STDERR "problem: empty dbinfo file\n"; + exit (1); + next; + } + + print "dbinfo: $file\n"; + + if ($retry) { + # use the dbinfo filename to get stage id, check if we have already updated this entry: + my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); + + # we just need to check for the existence of an entry: + if ($runData->{stage} eq "chip") { + my $cmd1 = "$chiptool -simple -processedimfile -chip_id $stage_id -class_id $class_id"; + $cmd1 .= " -dbname $dbname " if defined($dbname); + + my ( $success1, $error_code1, $full_buf1, $stdout_buf1, $stderr_buf1 ) = + run(command => $cmd1, verbose => 0); + + unless ($success1) { + $error_code1 = (($error_code1 >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run chiptool to check on results", $remote_id, $error_code1); + } + + my $Nrows = @$stdout_buf1; + if ($Nrows > 0) { + print STDERR "already updated: $stage_id, $class_id\n"; + next; + } + } elsif ($runData->{stage} eq "camera") { + my $cmd1 = "$camtool -simple -processedexp -cam_id $stage_id"; + $cmd1 .= " -dbname $dbname " if defined($dbname); + + my ( $success1, $error_code1, $full_buf1, $stdout_buf1, $stderr_buf1 ) = + run(command => $cmd1, verbose => 0); + + unless ($success1) { + $error_code1 = (($error_code1 >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run chiptool to check on results", $remote_id, $error_code1); + } + + my $Nrows = @$stdout_buf1; + if ($Nrows > 0) { + print STDERR "already updated: cam_id: $stage_id\n"; + next; + } + } elsif ($runData->{stage} eq "warp") { + my $cmd1 = "$warptool -simple -warped -warp_id $stage_id -skycell_id $class_id"; + $cmd1 .= " -dbname $dbname " if defined($dbname); + + my ( $success1, $error_code1, $full_buf1, $stdout_buf1, $stderr_buf1 ) = + run(command => $cmd1, verbose => 0); + + unless ($success1) { + $error_code1 = (($error_code1 >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run chiptool to check on results", $remote_id, $error_code1); + } + + my $Nrows = @$stdout_buf1; + if ($Nrows > 0) { + print STDERR "already updated: warp_id: $stage_id, $class_id\n"; + next; + } + } else { + print STDERR "do not know how to handle stage $runData->{stage}\n"; + exit (3); + } + } + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + # This shouldn't fail, but we also can't suddenly abort if something does fail. + # Now we're going to drop the component that owns this dbinfo, which should force a retry. + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); + my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); + $stage_fail{$stage_id} = 1; + } + } + } + print STDERR "dbinfo files all run (or skipped)\n"; +} # STEP 8: unify the stage states and update the remoteComponent entries @@ -412,10 +451,9 @@ } elsif ($stage eq 'camera') { - $stage_id =~ s/^.*cm\.//; + ($stage_id) = $file =~ /cm\.(\d*).dbinfo/; $class_id = 0; } elsif ($stage eq 'warp') { - $stage_id =~ s/^.*wrp\.//; - $class_id = 0; + ($stage_id, $class_id) = $file =~ /wrp\.(\d*)\.(skycell.\d\d\d\d.\d\d\d).dbinfo/; } elsif ($stage eq 'stack') {