#! /usr/bin/env perl

use DBI;
use PS::IPP::Config 1.01 qw( :standard );
use PS::IPP::Metadata::Config;
use PS::IPP::Metadata::List qw( parse_md_list );
use IPC::Cmd 0.36 qw( can_run run );
use Carp;
use Pod::Usage qw( pod2usage );

use Nebulous::Server::SQL;
use Nebulous::Client;
use SQL::Interp qw( sql_interp );
use strict;
use warnings FATAL => qw( all );
use Getopt::Long qw( GetOptions :config auto_help auto_version );
use File::Basename;


my ($host,$volume,$dir);
GetOptions(
    'host=s' => \$host,
    'volume=s' => \$volume,
    'dir=s'   => \$dir) || pod2usage(2 );

my (    $db, $dbhost, $dbpass, $dbuser);
my $conffile = "/home/panstarrs/ipp/ippconfig/nebulous.config";
if (-e $conffile) {
#    printf STDERR "found config\n";
    open(IN,$conffile);
    while(<IN>) {
        chomp;
        if ($_ =~ /NEB_DB\s+/) {
            $db = (split /\s+/,$_)[2];
        }
        elsif ($_ =~ /NEB_DBHOST/) {
            $dbhost = (split /\s+/,$_)[2];
        }
        elsif ($_ =~ /NEB_USER/) {
            $dbuser = (split /\s+/,$_)[2];
        }
        elsif ($_ =~ /NEB_PASS/) {
            $dbpass = (split /\s+/,$_)[2];
        }
    }
    close(IN);
#    print STDERR  "$db $dbhost $dbuser $dbpass\n";
}

pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless
    defined $host && defined $volume && defined $dir && 
    defined $db && defined $dbhost && defined $dbuser && defined $dbpass;

my $host_check = `hostname`;
chomp($host_check);
if ($host ne $host_check) {
    die "Required to run on $host.  not on $host_check\n";
}

my $dbh = DBI->connect(
    "DBI:mysql:database=$db:host=$dbhost",
    $dbuser,
    $dbpass,
    {
        RaiseError => 1,
        PrintError => 0,
        AutoCommit => 0,
    },
    );
my $sql = Nebulous::Server::SQL->new();

my $server = $ENV{'NEB_SERVER'};
my $neb = Nebulous::Client->new(
    proxy => "$server",
    );

my $time = time();
my $logDest = "neb://any/stsci_shuffle/${host}.${volume}/${dir}.log.${time}";
my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n";
$ipprc->redirect_output($logDest) or die "Could not redirect output to logfile ${logDest}\n";

my $unknown_dir = "/data/ippb06.0/stsci_unknown/";
my $root = "/data/${host}.${volume}/nebulous/${dir}";
my @subdirectories = <$root/*>;
my $md5_fails = 0;

#print "/data/${host}.${volume}/nebulous/${dir}/*\n";
#print "$#subdirectories\n";
foreach my $subdir (@subdirectories) {
    my @files = <${subdir}/*>;
    my $otime = $time;
    $time = time();
    my $dtime = $time - $otime;
    print "LOG BEGIN $subdir $time $dtime\n";
    
    my @belongs;
    my @keys;
    my @vols;
    my @md5sums;
    my @skip;
# Pre scan
    for (my $i = 0; $i <= $#files; $i++) {
	my $f = $files[$i];
	my ($ins_id, $belongs);
	$ins_id = $f;
	$ins_id =~ s/^$subdir\///;
	$ins_id =~ s/\..*//;

	$belongs[$i] = 0; 

	if ($ins_id > 4956576912) { 
	    if ($f =~ /LAP.PV3.20140730/) { # Only PV3
		if (($f =~ /wrp/)) {
		    if (($f =~ /fits/)||($f =~ /cmf/)||($f =~ /psf/)) {
			$belongs[$i] = 1;
		    }
		}
		if (($f =~ /stk/)&&($f =~ /fits/)&&($f =~ /unconv/)) {
		    $belongs[$i] = 1;
		}
	    }
	}
	
	if ($belongs[$i] == 0) {
	    $keys[$i] = $f;
	    $keys[$i] =~ s/.*\/${ins_id}\.//;
	    $keys[$i] =~ s%:%/%g;
	    # Compressed fits 
	    if ($keys[$i] =~ /.fits.fz$/) { 
		$keys[$i] =~ s/.fits.fz$/.fits/;
	    }

	    $md5sums[$i] = actual_local_md5sum($f);
	    unless (defined($md5sums[$i])) {
		die("Failed to calculate md5sum locally. $f ");
	    }
	    # Get a target
	    $vols[$i] = get_volume($keys[$i]);
	    if (defined($vols[$i])) {
		$vols[$i] = $vols[$i] . ".0";
	    }
	    
	    # Check that we actually have a key on the host we're checking.
	    my $local_uris = $neb->find_instances($keys[$i],"~${host}.${volume}");
	    if ($#{ $local_uris } == -1) { 
		$vols[$i] = undef();
		print "# No registered copy of $keys[$i] on ${host}.${volume}.  Treating as NOVOL\n";
#		die "No local copy of $keys[$i] on $host . $volume\n";
	    }
	}
	elsif ($belongs[$i] == 1) {
	    $md5sums[$i] = 'NOT_CALCULATED';
	    $vols[$i]    = 'ippXXX.Y';
	    $keys[$i]    = 'NO_REPLICATE';
	}
	if (defined($vols[$i])) {
	    print "## $ins_id $md5sums[$i] $vols[$i] $belongs[$i] $f $keys[$i] \n";
	}
	else {
	    print "## $ins_id $md5sums[$i] NOVOL_BECAUSE_UNKNOWN $belongs[$i] $f $keys[$i] \n";
	}
    }
    $otime = $time;
    $time = time();
    $dtime = $time - $otime;
    print "LOG SCAN $subdir $time $dtime\n";
#    die;
# Do a replication pass
    for (my $i = 0; $i <= $#files; $i++) {
	$skip[$i] = 0;
	if ($belongs[$i] == 0) {
	    if (defined($vols[$i])) {
		unless ($neb->replicate($keys[$i],$vols[$i])) {
		    $skip[$i] = 1;
		}
#		if ($@) { die $@; }
	    }
	    else {
		# Key wasn't defiend
		my $dir = dirname($keys[$i]);
		system("mkdir -p ${unknown_dir}/${dir}");
		system("cp -a $files[$i] ${unknown_dir}/$keys[$i]");
	    }
	}
    }
    system("sync") == 0 or die "Couldn't sync?";
    $otime = $time;
    $time = time();
    $dtime = $time - $otime;
    print "LOG REPL $subdir $time $dtime\n";

# Do the check pass
    for (my $i = 0; $i <= $#files; $i++) {
	if ($skip[$i] == 1) { next; }
	my $uri;
	if ($belongs[$i] == 0) {
	    if (defined($vols[$i])) {
		print "$keys[$i] $vols[$i] $host $volume \n";
		my $uris = $neb->find_instances($keys[$i],$vols[$i]);
		@$uris = map {URI->new($_)->file if $_} @$uris;
		$uri = ${ $uris }[0];
	    }
	    else {
		$uri = "${unknown_dir}/$keys[$i]";
	    }

	    my $tmpmd5 = local_md5sum($uri);
	    unless (defined($tmpmd5)) { 
		next; 
	    }
	    if ($tmpmd5 ne $md5sums[$i]) { 
		if (defined($vols[$i])) {
		    print "MD5SUM FAIL: $keys[$i] $vols[$i] $md5sums[$i] found: $tmpmd5 \n";
		}
		else {
		    print "MD5SUM FAIL: $keys[$i] NOVOL_BECAUSE_UNKNOWN $md5sums[$i] found: $tmpmd5 \n";
		}
		$md5_fails ++;
		next;
#		die "Post-replication md5sum does not match! $keys[$i]: $vols[$i] $tmpmd5 != $md5sums[$i]";
	    }
	    # End my best validation thought.	    

	    if (defined($vols[$i])) {
#		print "HERE: $keys[$i] $host $volume\n";
		eval {
		    $neb->cull($keys[$i],$host . "." . $volume);
		};
		warn "CULL ERROR: $@" if $@;
	    }
	    else {
		system("rm $files[$i]");
	    }		
	}
    }

    $otime = $time;
    $time = time();
    $dtime = $time - $otime;
    print "LOG END $subdir $time $dtime $md5_fails\n";
#    die;
}

sub get_volume {
    my $key = shift;


    my ($vol_id, $vol_host, $vol_path, $xattr, $forbidden_cabinet);    
    my $max_used_space = 0.98;
    my $topfew_count = 15;
    eval {
	my $rows;
        
	my $query = $dbh->prepare( $sql->get_cabinets_for_ext_id );
	$rows = $query->execute($key);
	unless ($rows > 0) {
	    $query->finish;
	    die("Requested key $key does not exist");
	}
	if ($rows == 1) {
	    ($forbidden_cabinet) = $query->fetchrow_array;
	    unless (defined($forbidden_cabinet)) {
		$forbidden_cabinet = 0;
	    }
	    $query->finish;
	}
	else {
	    $forbidden_cabinet = 0;
	    $query->finish;
	}
	
	
	$query = $dbh->prepare( $sql->get_replication_volume_for_ext_id );
	# ext_id, %free, avaiable, allocate
	$rows = $query->execute($key, $max_used_space, 1, 1, $forbidden_cabinet, $topfew_count);
	# XXX destinguish between non-existant and unaviable
	unless ($rows > 0) {
	    $query->finish;
	    die("can't find a suitable storage volume to replicate $key to");
	}
	# when matching by name we shouldn't ever match more than once
	if ($rows > 1) {
	    $query->finish;
	    die("affected row count is $rows instead of 1");
	}
	
	my $free;
	($vol_id, $vol_host, $vol_path, $xattr, $free) = $query->fetchrow_array;
#	print "$vol_id $vol_host $vol_path\n";
	$query->finish;
    };

    if (defined($vol_host)) {
	if ($vol_host eq 'ipp066') { 
	    $vol_host = get_volume($key); 
	    if ($vol_host eq 'ipp066') { 
		# Really?
	    	$vol_host = get_volume($key); 
#		if ($vol_host eq 'ipp066') {}  #fine.  Be that way.  Jerk.
#                 
	    }
	}
    }
    return($vol_host);
}
    

sub secure_replicull {
    my $key = shift;
    my $md5sum = shift;
    my $rep_vol = shift;
    my $del_vol = shift;

    $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
    if ($@) { die $@; }

    # Begin my best validation thought
    system("sync") == 0 or die "Couldn't sync?";
    my $uris = $neb->find_instances($key,$rep_vol);
    @$uris = map {URI->new($_)->file if $_} @$uris;
    my $tmpmd5 = local_md5sum(${ $uris }[0]);
    if ($tmpmd5 ne $md5sum) { 
	sleep(2);
	$tmpmd5 = local_md5sum(${ $uris }[0]);
	if ($tmpmd5 ne $md5sum) { 
	    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
	}
    }
    # End my best validation thought.
    
    $neb->cull($key,$del_vol);
}

sub local_md5sum {
    my $filename = shift;
    my $volume   = (split /\//, $filename)[2];
    my $host     = $volume;
    $host =~ s/\.\d//;
#    print "$filename $host $volume\n";
    my $response = `ssh $host md5sum $filename`;
    chomp($response);
    my ($sum, undef) = split /\s+/, $response;
     unless(defined($sum)) {
         warn("Failed to calculate md5sum locally. $filename $host $volume");
     }
    return($sum);
}

sub actual_local_md5sum {
    my $filename = shift;
    my $response = `md5sum $filename`;
    chomp($response);
    my ($sum, undef) = split /\s+/, $response;
     unless(defined($sum)) {
         warn("Failed to calculate md5sum Actually locally. $filename $host $volume");
     }
    return($sum);
}
