#!/usr/bin/env perl 

use Carp;
use warnings;
use strict;

use IPC::Cmd 0.36 qw( can_run run );
use PS::IPP::Metadata::Config;
use PS::IPP::Metadata::List qw( parse_md_list );
use PS::IPP::Config 1.01 qw( :standard );

use File::Basename;
use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
use Pod::Usage qw( pod2usage );

# Hard coded values
my $DMZ_HOST = 'wtrw';
my $SEC_HOST = 'mu-fe';
my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
my $remote_root  = '/scratch3/watersc1/';

# tools
my $missing_tools;
my $ssh        = can_run('ssh')  or (warn "Can't find ssh" and $missing_tools = 1);
my $scp        = can_run('scp')  or (warn "Can't find scp" and $missing_tools = 1);
my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);

if ($missing_tools) {
    warn("Can't find required tools.");
    exit($PS_EXIT_CONFIG_ERROR);
}

# Options
my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera);
$force = 0;
$verbose = 0;
GetOptions(
    'remote_id=s'   => \$remote_id,
    'job_id=s'      => \$job_id,
    'path_base=s'   => \$path_base,
    'policy=s'      => \$policy,
    'poll'          => \$poll,
    'force-poll'    => \$force,
    'camera=s'      => \$camera,
    'dbname=s'      => \$dbname,
    'verbose'       => \$verbose,
    'no_update'     => \$no_update,
    ) or pod2usage( 2 );

pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
pod2usage( -msg => "Required options: --remote_id --path_base", -exitval => 3) unless
    defined($path_base) and
    defined($remote_id);

my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id);
my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files                                    

# Phase 1: See if we can actually do anything.
# If the link is down, there's no benefit in trying to do anything else.
&check_ssh_connection();

print "passed authentication challenge.\n";

# Phase 1.5: Grab the information about this run;
my $rt_cmd = "$remotetool -listrun -remote_id $remote_id ";
$rt_cmd   .= " -dbname $dbname " if defined($dbname);
my ( $rt_success, $rt_error_code, $rt_full_buf, $rt_stdout_buf, $rt_stderr_buf ) =
    run(command => $rt_cmd, verbose => $verbose);
unless ($rt_success) {
    $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR);
    &my_die("UNable to run remotetool to determine remote run status",
	    $remote_id,$rt_error_code);
}
my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or 
    &my_die("UNable to run remotetool to determine remote run status",
	    $remote_id,$rt_error_code);
my $rrData2 = parse_md_list($rrData);
my $runData = ${ $rrData2 }[0]; # There should be only one

# These are operations we only need to do on the first call, not on subsequent poll operations.
unless ($poll) {
# Phase 2: Ensure files are in place:
# Copy command files
    my @files = ();
    
    my $uri_command = $path_base . ".cmd";
    my $uri_transfer= $path_base . ".transfer";
    my $uri_check   = $path_base . ".check";
    my $uri_config  = $path_base . ".config";
    my $uri_generate= $path_base . ".generate";
    my $uri_return  = $path_base . ".return";

    my $disk_command = $ipprc->file_resolve($uri_command);
    my $disk_transfer= $ipprc->file_resolve($uri_transfer);
    my $disk_check   = $ipprc->file_resolve($uri_check);
    my $disk_config  = $ipprc->file_resolve($uri_config);
    my $disk_generate= $ipprc->file_resolve($uri_generate);
    my $disk_return  = $ipprc->file_resolve($uri_return);
    
    scp_put($disk_command,uri_local_to_remote($uri_command));
    scp_put($disk_transfer,uri_local_to_remote($uri_transfer));
    scp_put($disk_check,uri_local_to_remote($uri_check));
    scp_put($disk_config,uri_local_to_remote($uri_config));
    scp_put($disk_generate,uri_local_to_remote($uri_generate));
    scp_put($disk_return, uri_local_to_remote($uri_return));
    
# Run check command
    my (undef,$remote_check) = uri_convert($uri_check);
    my (undef,$remote_transfer)= uri_convert($uri_transfer);

    my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer");
    # We no longer need to parse this output, as it retrieves files itself.
    foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) {
	print "$l\n";
    }

# Run real command
    my (undef,$remote_command) = uri_convert($uri_command);
    my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command");
    if ($#{ $ssh_exec_stdout } != -1) { # Parse the output
	$job_id = ${ $ssh_exec_stdout }[0];
	chomp($job_id);
	$job_id =~ s/\s+//g;
    }

    unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly.
	&my_die("No job_id returned.  Sorry.",$remote_id,$PS_EXIT_PROG_ERROR);
    }

    # Notify the database that this entry is currently running.
    my $command = "$remotetool -updaterun -remote_id $remote_id ";
    $command .= " -set_state run ";
    $command .= " -job_id $job_id ";
    $command .= " -dbname $dbname " if defined $dbname;
    
    system($command);
    
} # End of exec phase

# Check that we have a valid job_id, either from the command line, or from the msub command.
unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly.
    &my_die("No job_id returned.  Sorry.",$remote_id,$PS_EXIT_PROG_ERROR);
}


# Poll the job status?
my $poll_count = 0;
my $poll_max   = 1; # If it isn't finished now, it's unlikely to magically finish in the next five minutes.
my $poll_sleep = 60;
my $poll_response = 0;
my $poll_word  = "nothing";

if ($force) {
    $poll_response = 1;
    $poll_word     = "Completed";
}

while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)&&($force != 1)) {
    if ($verbose) {
	print "Polling code.  Sleeping.  $poll_count of $poll_max $poll_response\n";
    }
    unless ($poll_count == 0) {
	sleep($poll_sleep);
    }
    ($poll_response,$poll_word) = poll_job($job_id);

    if ($poll_response == 1) { # This job has completed
	last;
    }
    elsif ($poll_response == -1) { # This job has an error
	warn("The job exited incorrectly: $remote_id $job_id $poll_word.  Attempting to recover what we can.");
	$poll_response = 1; # To fool the check later into proceding
    }
    $poll_count++;
}
if ($verbose) {
    print "Stopped Polling code.  $poll_count of $poll_max $poll_response\n";
}
if ($poll_response != 1) {
    &my_die("Job hasn't completed.  Ending polling.", $remote_id, 0, "run", $job_id);
}


# Initialize the remote side to sync things

# Once sync completes, execute all the database commands
# Grab the return list
my $uri_return  = $path_base . ".return";
my ($ipp_return,$remote_return) = uri_convert($uri_return);
# We shouldn't need to get this file.  We made that file.
#scp_get($remote_return,$ipp_return);

# Construct list of dbinfo files we'll need to execute.
my @returned_files = ();
my @dbinfo_files   = ();
my @dbindex        = ();
my $iter           = 0;
open(RETURN,   "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id,$PS_EXIT_SYS_ERROR);
while (<RETURN>) {
    chomp;
    push @returned_files, $_;
    if ($_ =~ /dbinfo/) {
	push @dbinfo_files, $_;
	push @dbindex, $iter;
    }
    $iter++;

}
close(RETURN);
print STDERR "Expect $#dbinfo_files dbinfo files\n";

my $first = 0;
my $last  = $#dbinfo_files;
my $mid   = 0;
my $range = 20;  # How wide of a chunk to look through to catch threading drops
my $offset = 0;  # Offset to start from.

# Feed the return list into the transfer tool
# If this fails, we should go to an auth wait point.
if ($poll_word eq 'Completed') {  
    # We're complete.  All files should exist on disk
    my $iter = 0;
    my $transfer_success = 1;
    do {
	$transfer_success = 1;
	$iter++;
	check_ssh_connection(); 
	my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
	my $return_push_output_print = join "\n", @{ $return_push_output };
	if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; }
	print "$return_push_output_print\n";
    } while (($transfer_success != 1) &&  ($iter < 10));  # Try harder;
}
else {
    # Just try and fetch things, because we didn't complete correctly, it seems.
    my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
    my $return_push_output_print = join "\n", @{ $return_push_output };
    print "$return_push_output_print\n";
}

foreach my $file (@dbinfo_files) {
    open(DBF,"$file") || warn "Missing file $file\n";
    my $cmd = <DBF>;
    close(DBF);
    chomp($cmd);
#    print STDERR "$file $cmd\n";
    $cmd =~ s/-/ -/g; # This is just a safety check for missing space.
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
	run(command => $cmd, verbose => $verbose);
    unless ($success) {

	# This shouldn't fail, but we also can't suddenly abort if something does fail.  
	# Now we're going to drop the component that owns this dbinfo, which should force a retry.
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code");
	if ($poll_word ne 'Completed') {  # If the job claims it completed, then we're fighting the network.
	    drop_component($remote_id,$cmd,$file);
	}
    }

}

# We're done, so set the state and exit.
&my_die("Finished",$remote_id,0,"full");


# END PROGRAM


sub check_ssh_connection {
    my $cmd = "$ssh -O check $DMZ_HOST";

    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
	run(command => $cmd, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Authorization check failed.",$remote_id,0,'auth');
    }
}

sub scp_put {
    my $file = shift;
    my $destination = shift;
    my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";

    my $directory = dirname($destination);
    ssh_exec_command("mkdir -p $directory");

    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
	run(command => $cmd, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die(); 
    }
}

sub scp_get {
    my $destination = shift;
    my $file = shift;

    my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file ";

    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
	run(command => $cmd, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die(); 
    }
}

sub ssh_exec_command {
    my $cmd = shift;
    $cmd = "$ssh  $DMZ_HOST ssh  ${SEC_HOST} $cmd";
    print "EXEC: $cmd\n";
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
	run(command => $cmd, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	# If we failed, see if we failed due to authorization.
	check_ssh_connection(); 
	&my_die("Failed to execute command: >>$cmd<<"); 
    }
    return ($stdout_buf);
}    

sub poll_job {
    my $job_id = shift;
    my $response_array = ssh_exec_command("checkjob -v $job_id");
    my $response = join "\n", @$response_array;
    
    if ($response =~ /State: Running/) { 
	return(0,"Running");
    }
    elsif ($response =~ /State: Completed/) {
	return(1,"Completed");
    }
    elsif ($response =~ /State: Idle/) {
	return(-2,"Idle");
    }
    else {
	my $response_word = $response;
	$response_word =~ s/.*State: //;
	$response_word =~ s/\n.*//;
	return(-1,$response_word);
    }
}


sub uri_convert {
    my $neb_uri = shift;
    my $ipp_disk= $ipprc->file_resolve( $neb_uri );
    my $remote_disk = $ipp_disk;
    
    unless(defined($ipp_disk)) {
	my_die();
    }

    $remote_disk =~ s%^.*/%%;   # Remove nebulous path
    $remote_disk =~ s%^\d+\.%%; # Remove ins_id
    $remote_disk =~ s%:%/%g;    # Replace colons with directories
    $remote_disk = "${remote_root}/${remote_disk}";
    return($ipp_disk,$remote_disk);
}

sub uri_to_outputs {
    my $neb_uri = shift;
    my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );

#    print TRANSFER "$ipp_disk\n";
#    print CHECK    "$remote_disk\n";
    return($ipp_disk,$remote_disk);
}

sub uri_local_to_remote {
    # This needs to replace the nebulous tag with the remote root.
    my $local_uri = shift;
    $local_uri =~ s%^.*?/%%; # neb:/
    $local_uri =~ s%^.*?/%%; # /
    $local_uri =~ s%^.*?/%%; # @HOST@.0/
    my $remote_uri = "${remote_root}/" . $local_uri;

    return($remote_uri);
}
 
sub uri_remote_to_local {
    # This needs to replace the remote root directory with the nebulous tag.
    my $remote_uri = shift;
    $remote_uri =~ s%${remote_root}%%;
    my $local_uri  = "neb:///" . $remote_uri;
    
    return($local_uri);
}

sub drop_component {
    my $remote_id = shift;
    my $cmd = shift;
    my $file = shift;

    if ($cmd eq '') { 
	# Zero byte file.  This dbinfo was never constructed properly.
	
	# Steal the stage_id from the filename.  This isn't something I'm happy about.
	my $stage = $runData->{stage};
	my $stage_id = $file;
	if ($stage eq 'chip') { 
	    $stage_id =~ s/^.*ch\.//;
	}
	elsif ($stage eq 'camera') {
	    $stage_id =~ s/^.*cm\.//;
	}
	elsif ($stage eq 'warp') {
	    $stage_id =~ s/^.*wrp\.//;
	}
	elsif ($stage eq 'stack') {
	    $stage_id =~ s/^.*stk\.//;
	}
	$stage_id =~ s/\..*$//;
	
	# Drop this from the database
	my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id ";
	$drop_cmd   .= " -dbname $dbname " if defined($dbname);
	system($drop_cmd);
    }

}

sub my_die {
    my $msg = shift;
    my $id  = shift;
    my $exit_code = shift;
    my $exit_state = shift;
    my $jobid = shift;
    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;

    carp($msg);
    
    if (defined $id and not $no_update) {
	my $command = "$remotetool -updaterun -remote_id $id";
	$command .= " -fault $exit_code " if defined $exit_code;
	$command .= " -job_id $job_id " if defined $jobid;
	$command .= " -set_state $exit_state " if defined $exit_state;
	$command .= " -dbname $dbname " if defined $dbname;

	system($command);
    }

    exit($exit_code);
}

# Quick review:
# new -> pending -> run -> full
# auth
