#! /usr/bin/env perl

use DBI;
use Nebulous::Client;
use Nebulous::Server::SQL;
#use Nebulous::Util;
use Nebulous::Key qw( parse_neb_key parse_neb_volume );

use SQL::Interp qw( sql_interp );


use IPC::Cmd 0.36 qw( can_run run );
use PS::IPP::Metadata::Config;
use PS::IPP::Metadata::List qw( parse_md_list );
use PS::IPP::Config 1.01 qw( :standard );
use Carp;
use strict;
use warnings FATAL => qw( all );
use Getopt::Long qw( GetOptions :config auto_help auto_version );
use File::Basename;


# Set up all the standard things
my $missing_tools;
my $warptool    = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
my $difftool    = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
my $stacktool   = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);

if ($missing_tools) {
    warn("Can't find required tools.");
    exit($PS_EXIT_CONFIG_ERROR);
}

my $verbose = 0;
my $debug   = 0;
my ($stage,$stage_id,$dbname);
GetOptions(
    'stage=s' => \$stage,
    'stage_id=s' => \$stage_id,
    'dbname=s'  => \$dbname,
) || pod2usage(2 );


my $camera = 'GPC1';
my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set_up", $stage, $stage_id);
my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files

# Grab a volume to use for this pass.
my @volumes = ();
for (my $i = 0; $i <= 19; $i++) {
    for (my $j = 0; $j < 3; $j++) {
        push @volumes, sprintf("stsci%02d.%1d",$i,$j);
    }
}
my $rand_vol = $volumes[rand @volumes];

# Set up the logs
my $time = time();
my $logDest = "neb://any/stsci_shuffle/${stage}/${stage}.${stage_id}.log.${time}";

unless ($debug) {
    $ipprc->redirect_output($logDest) or &my_die("Could not redirect output to logfile ${logDest}", $stage,$stage_id);
}

my $manifest_dir = "/data/${rand_vol}/MANIFEST/${stage}";
system("mkdir -p ${manifest_dir}");
open(MANIFEST,">${manifest_dir}/${stage}.${stage_id}.MANIFEST") || &my_die( "Could not open manifest file", $stage,$stage_id);

# Set up the nebulous database connection.
my (    $db, $dbhost, $dbpass, $dbuser);
my $conffile = "/home/panstarrs/ipp/ippconfig/nebulous.config";
if (-e $conffile) {
#    printf STDERR "found config\n";
    open(IN,$conffile);
    while(<IN>) {
        chomp;
        if ($_ =~ /NEB_DB\s+/) {
            $db = (split /\s+/,$_)[2];
        }
        elsif ($_ =~ /NEB_DBHOST/) {
            $dbhost = (split /\s+/,$_)[2];
        }
        elsif ($_ =~ /NEB_USER/) {
            $dbuser = (split /\s+/,$_)[2];
        }
        elsif ($_ =~ /NEB_PASS/) {
            $dbpass = (split /\s+/,$_)[2];
        }
    }
    close(IN);
#    print STDERR  "$db $dbhost $dbuser $dbpass\n";
}

pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless
    defined $db && defined $dbhost && defined $dbuser && defined $dbpass;

my $dbh = DBI->connect(
    "DBI:mysql:database=$db:host=$dbhost",
    $dbuser,
    $dbpass,
    {
        RaiseError => 1,
        PrintError => 0,
        AutoCommit => 0,
    },
    );
my $sql = Nebulous::Server::SQL->new();

my $server = $ENV{'NEB_SERVER'};
my $neb = Nebulous::Client->new(
    proxy => "$server",
    );


# Define which things we want to look at, and where and how many of each.
my %components = ();
my $command = '';
if ($stage eq 'warp') {
    %components = ('PSWARP.OUTPUT' => { 'local' => 0, 'stsci' => 1 },
		   'PSWARP.OUTPUT.MASK' => { 'local' => 0, 'stsci' => 1 },
		   'PSWARP.OUTPUT.VARIANCE' => { 'local' => 0, 'stsci' => 1 },
		   'PSWARP.OUTPUT.SOURCES' => { 'local' => 2, 'stsci' => 1 },
		   'PSWARP.CONFIG' => { 'local' => 2, 'stsci' => 1}
	);
    $command = "$warptool -warped -warp_id $stage_id -dbname $dbname";
}
elsif ($stage eq 'stack') {
    %components = ('PPSTACK.UNCONV' => { 'local' => 2, 'stsci' => 1 },
		   'PPSTACK.UNCONV.MASK' => { 'local' => 2, 'stsci' => 1 },
		   'PPSTACK.UNCONV.VARIANCE' => { 'local' => 2, 'stsci' => 1 },
		   'PPSTACK.UNCONV.EXP' => { 'local' => 2, 'stsci' => 1 },
		   'PPSTACK.UNCONV.EXPNUM' => { 'local' => 2, 'stsci' => 1 },
		   'PPSTACK.UNCONV.EXPWT' => { 'local' => 2, 'stsci' => 1 },
		   'PSPHOT.PSF.SKY.SAVE' => { 'local' => 2, 'stsci' => 1 },
		   'PSPHOT.STACK.OUTPUT' => { 'local' => 2, 'stsci' => 1 },
		   'PPSTACK.CONFIG' => { 'local' => 2, 'stsci' => 1}

	);
    $command = "$stacktool -sumskyfile -stack_id $stage_id -dbname $dbname";
}
elsif ($stage eq 'diff') {
    %components = ('PPSUB.OUTPUT.KERNELS' => { 'local' => 2, 'stsci' => 1 },
		   'PPSUB.OUTPUT.SOURCES' => { 'local' => 2, 'stsci' => 1 },
		   'PPSUB.CONFIG' => { 'local' => 2, 'stsci' => 1 }
	);
    $command = "$difftool -diffskyfile -diff_id $stage_id -dbname $dbname";
}

# Get the stage information 
my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    run(command => $command, verbose => $verbose);
unless ($success) {
    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    &my_die("Unable to run STAGEtool -liststagestuff ", $stage,$stage_id);
}
my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to run STAGEtool -liststagestuff ", $stage,$stage_id);
my $stageData = parse_md_list($MDlist);

# Determine a file list;
my @keylist = ();


my $otime = $time;
$time = time();
my $dtime = $time - $otime;
log_print ("LOG BEGIN $stage $stage_id $rand_vol");

foreach my $stageEntry ( @{ $stageData } ) {
    my $quality   = $stageEntry->{quality};
    if ($quality != 0) { next; } # SKip things with bad quality.

    my $path_base = $stageEntry->{path_base};

    my %replicates;
    my %culls;
    my %md5sums_for_keys;
    my %canonical_files;
    foreach my $component (keys %components) {
	my $key;
	if (($stage eq 'stack')&&($component eq 'PSPHOT.STACK.OUTPUT')) {
	    # Do special things to get to skycalresult
	    my $ss_command = "$staticskytool -skycalresult -stack_id $stage_id -label LAP.PV3.20140730.skycal01 -dbname $dbname";
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to run STAGEtool -liststagestuff ", $stage,$stage_id);
	    }
	    my $SS_MDlist = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to run STAGEtool -liststagestuff ", $stage,$stage_id);
	    my $SS_data = parse_md_list($MDlist);
	    my $SSRun = ${ $SS_data }[0];
	    my $SS_pb = $SSRun->{path_base};
	    $key = $ipprc->filename($component,$SS_pb);
	}
	else {
	    $key = $ipprc->filename($component,$path_base);
	}

	# I hate you, nebulous.
	eval {
	    $key = parse_neb_key($key);
	    $key = $key->path;
	};
	


	# Stealing most of neb-stat here
	my %md5sum_uniq;
	my $instances = $neb->find_instances($key, 'any');
	my @files = map {URI->new($_)->file if $_} @$instances;
	my @stsci_copies = ();
	my @stsci_volumes = ();
	my @local_copies = ();
	my @local_volumes = ();
	my @existance;
	my @md5sums;
	my $local_count = 0;
	my $stsci_count = 0;

	for (my $i = 0; $i <= $#files; $i++) {
	    my $volume;
	    if (-e $files[$i]) {
		$existance[$i] = 1;
		($md5sums[$i],$volume) = local_md5sum($files[$i]);
		$md5sum_uniq{$md5sums[$i]} = 1;
		if ($volume =~ /stsci/) { 
		    unless (defined($canonical_files{$key})) {
			$canonical_files{$key} = $files[$i];
		    }
		    push @stsci_copies, $files[$i];
		    push @stsci_volumes, $volume;
		}
		elsif ($volume =~ /ipp/) { 
		    push @local_copies, $files[$i];
		    push @local_volumes, $volume;
		}
		else { &my_die("You can't exist somewhere not ipp and not stsci",$stage,$stage_id,$files[$i]); }
	    }
	}
	
	# Check that we have only one md5sum value.
	if (scalar(keys %md5sum_uniq) != 1) {
	    &my_die("Key $key has inconsistent md5sum values", $stage,$stage_id);
	}
	$md5sums_for_keys{$key} = (keys %md5sum_uniq)[0];

	# Do the logic at what we found
	if ($#stsci_copies + 1 < $components{$component}{stsci}) {
	    # We need to make an stsci copy.
	    push @{ $replicates{$key} }, $rand_vol;
	}
	if ($#local_copies + 1 < $components{$component}{local}) {
	    my $vol = get_volume($key);
	    push @{ $replicates{$key} }, $vol;
	}
	if ($#stsci_copies + 1 > $components{$component}{stsci}) {
	    for (my $i = $components{$component}{stsci}; $i <= $#stsci_copies; $i++) {
		push @{ $culls{$key} }, $stsci_volumes[$i]; # $stsci_copies[$i];
	    }
	}
	if ($#local_copies + 1 > $components{$component}{local}) {
	    for (my $i = $components{$component}{local}; $i <= $#local_copies; $i++) {
		push @{ $culls{$key} }, $local_volumes[$i]; # $local_copies[$i];
	    }
	}

	my $copy_check = sprintf("ST: %d %d IPP: %d %d",
				 $#stsci_copies +1,  $components{$component}{stsci}, 
				 $#local_copies + 1, $components{$component}{local});
	log_print("SCAN $stage $stage_id $path_base $component $copy_check");
    }


    if ($debug) { 
	foreach my $key (keys %replicates) {
	    foreach my $vol (@{ $replicates{$key} }) {
		log_print("REP: $key to $vol");
	    }
	}
	foreach my $key (keys %culls) {
	    foreach my $vol (@{ $culls{$key} }) {
		log_print("CULL: $key from $vol");
	    }
	}
	die;
    }
    
    foreach my $key (keys %replicates) {
	foreach my $vol (@{ $replicates{$key} }) {
	    log_print("REP: $key to $vol");
	    $neb->replicate($key,$vol) or &my_die( "failed to replicate a copy of $key to $vol");
	    if ($@) { die $@; }
	}
    }
    system("sync") == 0 or die "Couldn't sync?";
    log_print("REPL $stage $stage_id $path_base");

    foreach my $key (keys %replicates) {
	foreach my $vol (@{ $replicates{$key} }) {
	    my $uris = $neb->find_instances($key,$vol);
	    @$uris = map {URI->new($_)->file if $_} @$uris;
	    my $uri = ${ $uris }[0];
	    my ($tmpmd5,undef) = local_md5sum($uri);
	    if ($tmpmd5 ne $md5sums_for_keys{$key}) {
		&my_die("Post-replication md5sum does not match for $key on $vol! $tmpmd5 != $md5sums_for_keys{$key}");
	    }
	    if ($vol =~ /stsci/) {
		$canonical_files{$key} = $uri;
	    }
	}

    }


    foreach my $key (keys %canonical_files) {
	print MANIFEST "$key $canonical_files{$key} $md5sums_for_keys{$key}\n";
    }

    log_print("CHCK $stage $stage_id $path_base");
    foreach my $key (keys %culls) {
	foreach my $vol (@{ $culls{$key} }) {
	    log_print("CULL $key from $vol");
	    $neb->cull($key,$vol);
	}
    }
    log_print("STOP $stage $stage_id $path_base");
}

close(MANIFEST);    
log_print("END $stage $stage_id ${manifest_dir}/${stage}.${stage_id}.MANIFEST");    

sub my_die {
    my $message = shift;
    carp($message);
    my $exit_code = $PS_EXIT_PROG_ERROR;
    close(MANIFEST);
    unlink("${manifest_dir}/${stage}.${stage_id}.MANIFEST");

    exit($exit_code);
}
	
sub log_print {
    my $msg = shift;

    $time = time();
    $dtime = $time - $otime;
    print "LOG $time $dtime $msg\n";
}

sub get_volume {
    my $key = shift;
    
    my ($vol_id, $vol_host, $vol_path, $xattr, $forbidden_cabinet);    
    my $max_used_space = 0.98;
    my $topfew_count = 25;
    eval {
	my $rows;
        
	my $query = $dbh->prepare( $sql->get_cabinets_for_ext_id );
	$rows = $query->execute($key);
	unless ($rows > 0) {
	    $query->finish;
	    die("Requested key $key does not exist");
	}
	if ($rows == 1) {
	    ($forbidden_cabinet) = $query->fetchrow_array;
	    unless (defined($forbidden_cabinet)) {
		$forbidden_cabinet = 0;
	    }
	    $query->finish;
	}
	else {
	    $forbidden_cabinet = 0;
	    $query->finish;
	}
	
	
	$query = $dbh->prepare( $sql->get_replication_volume_for_ext_id );
	# ext_id, %free, avaiable, allocate
	$rows = $query->execute($key, $max_used_space, 1, 1, $forbidden_cabinet, $topfew_count);
	# XXX destinguish between non-existant and unaviable
	unless ($rows > 0) {
	    $query->finish;
	    die("can't find a suitable storage volume to replicate $key to");
	}
	# when matching by name we shouldn't ever match more than once
	if ($rows > 1) {
	    $query->finish;
	    die("affected row count is $rows instead of 1");
	}
	
	my $free;
	($vol_id, $vol_host, $vol_path, $xattr, $free) = $query->fetchrow_array;
#	print "$vol_id $vol_host $vol_path\n";
	$query->finish;
    };
    &my_die($@) if $@;

    if (defined($vol_host)) {
	if ($vol_host eq 'ipp066') { 
            $vol_host = get_volume($key); 
            if ($vol_host eq 'ipp066') { 
                # Really?
                $vol_host = get_volume($key); 
#               if ($vol_host eq 'ipp066') {}  #fine.  Be that way.  Jerk.
#                 
            }
        }
    }

    return($vol_host . ".0");
}
    
sub local_md5sum {
    my $filename = shift;
    my $volume   = (split /\//, $filename)[2];
    my $host     = $volume;
    $host =~ s/\.\d//;
#    print "$filename $host $volume\n";
    my $response = `ssh -x $host md5sum $filename`;
    chomp($response);
    my ($sum, undef) = split /\s+/, $response;
    unless(defined($sum)) {
        die("Failed to calculate md5sum locally. $filename $host $volume");
    }
    return($sum,$volume);
}
