#!/usr/bin/env perl

use strict;
use warnings FATAL => qw( all );

use vars qw( $VERSION );
$VERSION = '0.01';

use Nebulous::Client;

use Getopt::Long qw( GetOptions :config auto_help auto_version );
use Pod::Usage qw( pod2usage );
use IPC::Cmd 0.36 qw( can_run run);
use PS::IPP::Metadata::List qw( parse_md_list );
use PS::IPP::Config 1.01 qw( :standard );

use Digest::MD5;
use URI;

my $missing_tools = 0;
#my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);

my ($server,$dbname,$stage,$stage_id);

$server = $ENV{'NEB_SERVER'} unless $server;
# $dbname = 'gpc1'; 
GetOptions(
    'server|s=s'     => \$server,
    'dbname=s'       => \$dbname,
    'stage=s'        => \$stage,
    'stage_id|x=s'     => \$stage_id,
) || pod2usage( 2 );

# Option parsing
pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
pod2usage( -msg => "Required options: --server", -exitval => 2 )
    unless $server;
pod2usage( -msg => "Required options: --dbname", -exitval => 2 )
    unless $dbname;
pod2usage( -msg => "Required options: --stage [camera|stack|skycal]", -exitval => 2 )
    unless $stage;
pod2usage( -msg => "missing key", exitval => 2 )
    unless defined $stage_id;

# Global options:

my $do_ops = 1;
my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1,
		    'ippb02' => 1, 'ippb03' => 1,
		    'ippb04' => 1, 'ippb05' => 1,
		    'ippb06' => 1
    );
my %backup_destinations = (#'ippb04' => 1,
			   #'ippb05' => 1,
			   'ippb06' => 1
);
my $backup_Nvols = 3;

my $ipprc;
if ($dbname eq 'gpc1') {
    $ipprc = PS::IPP::Config->new( "GPC1" );
}
else {
    die "Unknown camera to use.";
}

# Set up nebulous db interface
my $neb = Nebulous::Client->new(
    proxy => "$server",
);
die "can't connected to Nebulous Server: $server"
    unless defined $neb;

# Pull data from the gpc1 database
my $verbose = 0;
my $mdcParser = PS::IPP::Metadata::Config->new;

my $files;
if ($stage eq 'camera') {
    my $cmd = "$camtool -processedexp -cam_id $stage_id -dbname $dbname";
    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
	run(command => $cmd, verbose => 0);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform stagetool: $error_code", $stage_id);
    }
    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
	&my_die("Unable to parse metadata from stagetool", $stage_id);
}
elsif ($stage eq 'stack') {
    my $cmd = "$stacktool -sumskyfile -stack_id $stage_id -dbname $dbname";
    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
	run(command => $cmd, verbose => 0);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform stagetool: $error_code", $stage_id);
    }
    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
	&my_die("Unable to parse metadata from stagetool", $stage_id);
}
elsif ($stage eq 'skycal') {
    my $cmd = "$staticskytool -skycalresult -skycal_id $stage_id -dbname $dbname";
    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
	run(command => $cmd, verbose => 0);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform stagetool: $error_code", $stage_id);
    }
    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
	&my_die("Unable to parse metadata from stagetool", $stage_id);
}

my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'],
		  'stack'  => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP',
			       'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'],
		  'skycal' => ['PSASTRO.OUTPUT.CMF']);


foreach my $entry (@$files) {
    my $path_base = $entry->{path_base};
    my $data_state = $entry->{state};
    my $hostname = $entry->{hostname};
    foreach my $product (@{ $components{$stage} }) {
	my @keys = ();
	if (($stage eq 'camera')&&($product eq 'PSASTRO.OUTPUT.MASK')) {
	    my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06',
			'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17',
			'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27',
			'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37',
			'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47',
			'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57',
			'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67',
			'XY71','XY72','XY73','XY74','XY75','XY76');
	    foreach my $ota (@otas) {
		push @keys, $ipprc->filename($product,$path_base,$ota);
	    }	    
	}
	else {
	    push @keys, $ipprc->filename($product,$path_base);
	}
# Do validation
	
	foreach my $key (@keys) {
	    # neb-stat level handling
	    my $stat = $neb->stat($key);
	    die "nebulous key: $key not found" unless $stat;   
	    my $instances = $neb->find_instances($key, 'any');
	    die "no instances found" unless $instances;   
	    
	    my $user_copies;
	    eval {
		$user_copies = $neb->getxattr($key, "user.copies");
	    };
	    unless(defined($user_copies)) {
		$user_copies = 1;
	    }
	    
	    my $md5sum;
	    my @validation;
	    my %md5sum_uniq;
	    my $existing_copies = 0;
	    
	    my @existance;
	    my @md5sums;
	    my @diskfiles = map {URI->new($_)->file if $_} @$instances;
	    my @diskvols;
	    my @diskhosts;
	    my @quality;
	    my $Ngood = 0;
	    my $quality_mask = 0;
	    
	    for (my $i = 0; $i <= $#diskfiles; $i++) {
		if (-e $diskfiles[$i]) {
		    $existance[$i] = 1;
		    $existing_copies++;
		    $md5sums[$i] = local_md5sum($diskfiles[$i]);
		    $md5sum_uniq{$md5sums[$i]} = 1;
		}
		else {
		    $existance[$i] = 0;
		    $md5sums[$i] = 'NON-EXISTANT';
		    $md5sum_uniq{$md5sums[$i]} = 1;
		}
		($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]);
		$validation[$i] = sprintf("% 3d %32s %s %s %d",
					  $existance[$i],
					  $md5sums[$i],
					  $diskfiles[$i],
					  $diskhosts[$i],$diskvols[$i]
		    );
		
		# Pre-parse decisions
		if ($existance[$i] == 0) {
		    $quality[$i] = 0;
		}
		elsif (is_backup_volume($diskhosts[$i])) {
		    $quality[$i] = 1;
		    $quality_mask = $quality_mask | 1;
		    $Ngood++;
		}
		else {
		    $quality[$i] = 2;
		    $Ngood++;
		}
	    }
	    if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth.
		die "There are multiple md5sum values for $key";
	    }
	    $md5sum = (keys(%md5sum_uniq))[0];
	    
	    print "\n$key $data_state $md5sum $hostname\n";
	    if (1) {
		
		print
		    "object id:             ", @$stat[0], "\n",
		    "key:                   ", @$stat[1], "\n";
		print 
		    "epoch:                 ", @$stat[4], "\n";
		print 
		    "md5sum count:          ", scalar(keys %md5sum_uniq), "\n";
		print 
		    "requested instances:   ", $user_copies, "\n",
		    "available instances:   ", @$stat[6], "\n",
		    "existing instances:    ", $existing_copies, "\n",
		    "total instances:       ", @$stat[7], "\n",
		    "instance location:\n", " " x 4;
		print
		    join("\n" . " " x 4, @validation), "\n";
	    }
	    
	    # Decide what to do
	    if ($Ngood == 0) {
		# DO something to attempt to fix this.
		my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/;
		
		open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |");
		my $good_file = '';
		while (<DD>) {
		    $_ =~ s/^\s+//;
		    my ($z,undef,$ff) = split /\s+/;
		    if (($ff)&&(-e $ff)) {
			my $md_response = `md5sum $ff`;
			if ($md_response =~ /$md5sum/) {
			    $good_file = (split /\s+/,$md_response)[1];
			}
		    }
		}
		close(DD);
		if ($good_file eq '') {
		    die "No valid instance of key: $key";
		}
		else {
		    $quality[0] = 1;
		    print "cp $good_file $diskfiles[0]\n";
		    system("cp $good_file $diskfiles[0]");
		}
		# Begin my best validation thought
		{
		    my $tmpmd5 = local_md5sum($diskfiles[0]);
		    if ($tmpmd5 ne $md5sum) { 
			die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
		    }
		}
		# End my best validation thought.
		
		$Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
	    }
	    
	    if ($quality[0] == 0) { # The first instance is bad.
		# But since we're here, and not up there, there must be at least one good copy.  Find it.
		for (my $i = 0; $i <= $#md5sums; $i++) {
		    if ($md5sums[$i] eq $md5sum) { # Found it.
			print "cp $diskfiles[$i] $diskfiles[0]\n";
			system("cp $diskfiles[$i] $diskfiles[0]");
			# Begin my best validation thought
			{
			    my $tmpmd5 = local_md5sum($diskfiles[0]);
			    if ($tmpmd5 ne $md5sum) { 
				die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
			    }
			}
			# End my best validation thought.
			last; # We're done here now.
		    }
		}	
	    }
	    
	    if ($Ngood == 1) { # We have only one version
		if ($quality_mask & 1) { # ANd it's on a backup volume
		    print "neb-replicate $key\n";
		    if ($do_ops) {
			$neb->replicate($key) or die "failed to replicate the single valid copy";
			if ($@) { die $@; }
		    }
		}
		else { # And it's not, so put one there
		    my $rep_vol = get_random_backup_volume();
		    print "neb-replicate --volume $rep_vol  $key\n";
		    if ($do_ops) {
			$neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node";
			if ($@) { die $@; }
			
			# Begin my best validation thought
			system("sync") == 0 or die "Couldn't sync?";
			my $uris = $neb->find_instances($key,$rep_vol);
			@$uris = map {URI->new($_)->file if $_} @$uris;
			my $tmpmd5 = local_md5sum(${ $uris }[0]);
			if ($tmpmd5 ne $md5sum) { 
			    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
			}
			# End my best validation thought.
		    }	    
		}
	    }
	    else { # N >= 2
		if (!($quality_mask & 1)) { # no backup copy
		    my $rep_vol = get_random_backup_volume();
		    print "neb-replicate --volume $rep_vol  $key\n";
		    if ($do_ops) {
			$neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
			if ($@) { die $@; }
			
			# Begin my best validation thought
			system("sync") == 0 or die "Couldn't sync?";
			my $uris = $neb->find_instances($key,$rep_vol);
			@$uris = map {URI->new($_)->file if $_} @$uris;
			my $tmpmd5 = local_md5sum(${ $uris }[0]);
			if ($tmpmd5 ne $md5sum) { 
			    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
			}
			# End my best validation thought.
		    }	    
		}
		if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid.
		    for (my $i = 0; $i <= $#diskfiles; $i++) {
			if ($quality[$i] == 2) {
			    $quality[$i] = 1;
			    last;
			}
		    }
		}
		for (my $i = 0; $i <= $#diskfiles; $i++) {
#	    print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
		    if ($quality[$i] != 1) {
			if ($existance[$i] == 0) { # This disk file doesn't exist.
			    system("touch $diskfiles[$i]");
			}
			my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
			print "neb-cull --volume $cull_vol $key\n";
			if ($do_ops) {
			    # The tilde here is to force hard volumes.  Don't touch it.
			    $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
			    if ($@) { die "$@"; }
			}
		    }
		}
	    }
	} # end keys for this product
    } # end product for this entry
} # end entry for this id

sub local_md5sum {
    my $filename = shift;
    my $volume   = (split /\//, $filename)[2];
    my $host     = $volume;
    $host =~ s/\.\d//;
#    print "$filename $host $volume\n";
    my $response = `ssh $host md5sum $filename`;
    chomp($response);
    my ($sum, undef) = split /\s+/, $response;
    unless(defined($sum)) {
	my_die("Failed to calculate md5sum locally. $filename $host $volume");
    }
    return($sum);
}


sub parse_volume {
    my $filename = shift(@_);
    my $full_volume   = (split /\//, $filename)[2];
    my ($hostname,$vol_index) = split /\./, $full_volume; # /;
    return($hostname,$vol_index);
}

sub is_backup_volume {
    my $hostname = shift(@_);
    if (exists($backup_hosts{$hostname})) {
	return(1);
    }
    return(0);
}

sub get_random_backup_volume {
    my $NN = scalar keys %backup_destinations;
    my $backup_host = (keys %backup_destinations)[int(rand($NN))];
    my $backup_vol  = int(rand($backup_Nvols));
    
    return("${backup_host}.${backup_vol}");
}

sub my_die {
    my $msg = shift(@_);
    print $msg . "\n";
    foreach my $a (@_) {
	print "ARG: $a\n";
    }
    die;
}




















__END__


