Index: /tags/ipp-pv3-20140717/ippScripts/Build.PL =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/Build.PL (revision 37264) +++ /tags/ipp-pv3-20140717/ippScripts/Build.PL (revision 37265) @@ -142,4 +142,5 @@ scripts/sc_prepare_warp.pl scripts/sc_remote_exec.pl + scripts/sc_retry_return.pl scripts/sc_transfer_tool.pl )], Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_retry_return.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_retry_return.pl (revision 37264) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_retry_return.pl (revision 37265) @@ -25,4 +25,5 @@ my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); if ($missing_tools) { @@ -32,5 +33,5 @@ # Options -my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera); +my ($remote_id,$path_base,$job_id,$dbname,$verbose,$no_update,$camera); $force = 0; $verbose = 0; @@ -39,7 +40,4 @@ 'job_id=s' => \$job_id, 'path_base=s' => \$path_base, - 'policy=s' => \$policy, - 'poll' => \$poll, - 'force-poll' => \$force, 'camera=s' => \$camera, 'dbname=s' => \$dbname, @@ -80,65 +78,4 @@ my $rrData2 = parse_md_list($rrData); my $runData = ${ $rrData2 }[0]; # There should be only one - -# keep this for reference for the moment, but this block should not be in this function -if (0) { -# Phase 2: Ensure files are in place: -# Copy command files - my @files = (); - - my $uri_command = $path_base . ".cmd"; - my $uri_transfer= $path_base . ".transfer"; - my $uri_check = $path_base . ".check"; - my $uri_config = $path_base . ".config"; - my $uri_generate= $path_base . ".generate"; - my $uri_return = $path_base . ".return"; - - my $disk_command = $ipprc->file_resolve($uri_command); - my $disk_transfer= $ipprc->file_resolve($uri_transfer); - my $disk_check = $ipprc->file_resolve($uri_check); - my $disk_config = $ipprc->file_resolve($uri_config); - my $disk_generate= $ipprc->file_resolve($uri_generate); - my $disk_return = $ipprc->file_resolve($uri_return); - - scp_put($disk_command,uri_local_to_remote($uri_command)); - scp_put($disk_transfer,uri_local_to_remote($uri_transfer)); - scp_put($disk_check,uri_local_to_remote($uri_check)); - scp_put($disk_config,uri_local_to_remote($uri_config)); - scp_put($disk_generate,uri_local_to_remote($uri_generate)); - scp_put($disk_return, uri_local_to_remote($uri_return)); - -# Run check command - my (undef,$remote_check) = uri_convert($uri_check); - my (undef,$remote_transfer)= uri_convert($uri_transfer); - - my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer"); - # We no longer need to parse this output, as it retrieves files itself. - foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) { - print "$l\n"; - } - -# Run real command - my (undef,$remote_command) = uri_convert($uri_command); - my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command"); - if ($#{ $ssh_exec_stdout } != -1) { # Parse the output - $job_id = ${ $ssh_exec_stdout }[0]; - chomp($job_id); - $job_id =~ s/\s+//g; - } - - unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. - &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); - } - - # Notify the database that this entry is currently running. - my $command = "$remotetool -updaterun -remote_id $remote_id "; - $command .= " -set_state run "; - $command .= " -job_id $job_id "; - $command .= " -dbname $dbname " if defined $dbname; - - print STDERR "$command\n"; - system($command); - -} # End of exec phase # Poll the job status? @@ -203,21 +140,50 @@ print "$return_push_output_print\n"; } while (($transfer_success != 1) && ($iter < 10)); # Try harder; + + if (!$transfer_success) &my_die ("failed to get all return files"); } foreach my $file (@dbinfo_files) { - # use the filename to get the stage id, check if we have already updated this entry: - - + + # use the dbinfo filename to get stage id, check if we have already updated this entry: + my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); + + # chiptool -dbname -chip_id $stage_id -class_id $class_id + + # we just need to check for the existence of an entry: + if ($runData->{stage} == "chip") { + my $cmd = "$chiptool -simple -processedimfile -chip_id $stage_id -class_id $class_id"; + $cmd .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run chiptool to check on results", $remote_id,$error_code); + } + + my $Nrows = @$stdout_buff; + if ($Nrows > 0) { + print STDERR "skip this entry: $stage_id, $class_id\n"; + next; + } + } open(DBF,"$file") || warn "Missing file $file\n"; my $cmd = ; close(DBF); + chomp($cmd); -# print STDERR "$file $cmd\n"; + $cmd =~ s/-/ -/g; # This is just a safety check for missing space. + if ($cmd == "") { + print STDERR "empty dbinfo file for $stage_id, $class_id, skipping\n"; + next; + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $cmd, verbose => $verbose); unless ($success) { - # This shouldn't fail, but we also can't suddenly abort if something does fail. # Now we're going to drop the component that owns this dbinfo, which should force a retry. @@ -228,5 +194,4 @@ } } - } @@ -234,8 +199,7 @@ &my_die("Finished",$remote_id,0,"full"); - # END PROGRAM - +# yes sub check_ssh_connection { my $cmd = "$ssh -O check $DMZ_HOST"; @@ -245,35 +209,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Authorization check failed.",$remote_id,0,'auth'); - } -} - -sub scp_put { - my $file = shift; - my $destination = shift; - my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}"; - - my $directory = dirname($destination); - ssh_exec_command("mkdir -p $directory"); - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die(); - } -} - -sub scp_get { - my $destination = shift; - my $file = shift; - - my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file "; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die(); + &my_die("Authorization check failed", $remote_id, 0, 'auth'); } } @@ -294,75 +228,49 @@ } +sub uri_convert { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk, $remote_disk); +} + sub poll_job { my $job_id = shift; my $response_array = ssh_exec_command("checkjob -v $job_id"); my $response = join "\n", @$response_array; - - if ($response =~ /State: Running/) { - return(0,"Running"); + + if ($response =~ /State: Running/) { + return(0,"Running"); } elsif ($response =~ /State: Completed/) { - return(1,"Completed"); + return(1,"Completed"); } elsif ($response =~ /State: Idle/) { - return(-2,"Idle"); + return(-2,"Idle"); } else { - my $response_word = $response; - $response_word =~ s/.*State: //; - $response_word =~ s/\n.*//; - return(-1,$response_word); - } -} - - -sub uri_convert { - my $neb_uri = shift; - my $ipp_disk= $ipprc->file_resolve( $neb_uri ); - my $remote_disk = $ipp_disk; - - unless(defined($ipp_disk)) { - my_die(); - } - - $remote_disk =~ s%^.*/%%; # Remove nebulous path - $remote_disk =~ s%^\d+\.%%; # Remove ins_id - $remote_disk =~ s%:%/%g; # Replace colons with directories - $remote_disk = "${remote_root}/${remote_disk}"; - return($ipp_disk,$remote_disk); -} - -sub uri_to_outputs { - my $neb_uri = shift; - my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - -# print TRANSFER "$ipp_disk\n"; -# print CHECK "$remote_disk\n"; - return($ipp_disk,$remote_disk); -} - -sub uri_local_to_remote { - # This needs to replace the nebulous tag with the remote root. - my $local_uri = shift; - $local_uri =~ s%^.*?/%%; # neb:/ - $local_uri =~ s%^.*?/%%; # / - $local_uri =~ s%^.*?/%%; # @HOST@.0/ - my $remote_uri = "${remote_root}/" . $local_uri; - - return($remote_uri); -} - -sub uri_remote_to_local { - # This needs to replace the remote root directory with the nebulous tag. - my $remote_uri = shift; - $remote_uri =~ s%${remote_root}%%; - my $local_uri = "neb:///" . $remote_uri; - - return($local_uri); -} + my $response_word = $response; + $response_word =~ s/.*State: //; + $response_word =~ s/\n.*//; + return(-1,$response_word); + } +} + # assumes $runData->{stage} exists sub get_stage_id_from_dbinfo { my $file = shift; + + my $stage_id = 0; + my $class_id = 0; # Steal the stage_id from the filename. This isn't something I'm happy about. @@ -373,10 +281,13 @@ elsif ($stage eq 'camera') { $stage_id =~ s/^.*cm\.//; + $class_id = 0; } elsif ($stage eq 'warp') { $stage_id =~ s/^.*wrp\.//; + $class_id = 0; } elsif ($stage eq 'stack') { $stage_id =~ s/^.*stk\.//; + $class_id = 0; } @@ -439,6 +350,2 @@ exit($exit_code); } - -# Quick review: -# new -> pending -> run -> full -# auth