#!/usr/bin/perl

$dumpspace = "/pallas/d1/";
$workspace = "/metis/d27/workspace/";
# $workspace = "/irene/d11/workspace/";

$scriptdir = "/hebe/d27/logs/";
$donelogfile = $scriptdir . "done.log";
$scriptfile = $scriptdir . "maestro.dat";
$flatdir = "/hebe/d27/references/config/";

$database = "/hebe/d27/database/";
$lockfile = $database . "lock";
$alockfile = $database . "alock";
$clockfile = $database . "clock";
$killfile  = $database . "kill";
$haltfile  = $database . "halt";
$reffile = $database . "start.clean";

sub vsystem {
    print STDERR "@_\n";
    $status = system ("@_");
    $status;
}

sub goodbye {
    print STDERR "ending execution\n";
    vsystem ("date");
    vsystem ("notify.pl $ARGV[0] none 0");
    die "@_";
}

if (@ARGV < 1) {die "USAGE: archive.pl Noutfile\n"; }
$Noutfile = $ARGV[0] + 1;
$outfile = "archive" . $Noutfile . ".log";

print STDERR "starting execution\n";
vsystem ("date");
if (-e $killfile) { goodbye "process killed with $killfile"; }


if (-e $alockfile) { goodbye "archive lock $alockfile is set"; }
open (TEMP, ">$alockfile");
close (TEMP);

# load in done date list
open (DONELOG, $donelogfile);
@donelist = ();
while ($line = <DONELOG>) {
  chop ($line);
  @donelist = (@donelist,$line);
}
$Ndone = @donelist;
close (DONELOG);

# load in maestro list
open (MAESTRO, $scriptfile);
@todolist = ();
while ($line = <MAESTRO>) {
  chop ($line);
  @todolist = (@todolist,$line);
}
$Ntodo = @todolist;
close (MAESTRO);

# find entries in MAESTRO that are NOT in DONELOG
@leftlist = ();
for ($i = 0; $i < $Ntodo; $i ++) {
    $gotit = 0;
    for ($j = 0; ($j < $Ndone) && ! $gotit; $j ++) {
	@entries = split (" ",$todolist[$i]);
	if ($entries[0] eq $donelist[$j]) {
	    $gotit = 1;
	}
    }
    if (!$gotit) {
	@leftlist = (@leftlist, $todolist[$i]);  
    }
}
$Nleft = @leftlist;

if (-e $killfile) { goodbye "process killed with $killfile"; }
# run controller on unfinished entries
# we are going to run this in the script directory
if ($Nleft) {
    chdir $scriptdir;
    for ($i = 0; $i < $Nleft; $i++) {
	print STDERR "unfinished: $leftlist[$i]\n";
	@entries = split (" ",$leftlist[$i]);
	$inlist = $entries[0] . ".inlist";
	$outlist = $entries[0] . ".outlist";
	$subdir = $entries[0];
	$photcode = $entries[1];
	print STDERR "$subdir: $inlist $outlist $photcode\n";
	if (-e $outlist) {
	    # partially successful run, restart controller in the middle
	    $tmp_inlist = $entries[0] . ".tmp_inlist";
	    $tmpoutlist = $entries[0] . ".tmpoutlist";
	    if (-e $tmpoutlist) { if (vsystem ("cat $tmpoutlist >> $outlist")) { goodbye "trouble merging $tmpoutlist and $outlist\n"; }}
	    # partially successful re-start: make sure $outlist is complete
	    if (parse_outlist ($outlist,$tmp_inlist,1)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s flatten", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    if (parse_outlist ($outlist,$tmp_inlist,2)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s dophot", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    if (parse_outlist ($outlist,$tmp_inlist,3)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s fstat", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    if (parse_outlist ($outlist,$tmp_inlist,4)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s gastro", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    if (parse_outlist ($outlist,$tmp_inlist,5)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s addstar", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    # do inlist entries not in outlist
	    if (parse_inlist ($inlist,$outlist,$tmp_inlist)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s flatten", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    unless (unlink ($tmpoutlist)) { goodbye "can't unlink $tmpoutlist, why not?"; }
	    unless (unlink ($tmp_inlist)) { goodbye "can't unlink $tmp_inlist, why not?"; }
	    vsystem ("cleaning.pl");
	    if (-e $lockfile) { goodbye "lock file exists"; }

	    print STDERR "done: $entries[0]\n";
	    open (DONELOG, ">>$donelogfile");
	    print DONELOG "$entries[0]\n";
	    close (DONELOG);
	} else {
	    $controlflags = "";
	    docontroller ($subdir, $inlist, $outlist, $photcode, "-s flatten", 1);
	    print STDERR "done: $subdir\n";
	    open (DONELOG, ">>$donelogfile");
	    print DONELOG "$subdir\n";
	    close (DONELOG);
	}
    }
    unless (unlink ($alockfile)) { goodbye "can't unlink $alockfile, why not?"; }
    print STDERR "remember to delete images from complete directories\n";

    unless (-e $haltfile) { 
	print STDERR "re-did unfinished images\n";
	print STDERR "starting a new archive.pl with output to $outfile\n";
	vsystem ("notify.pl $ARGV[0] $outfile $Noutfile");
	vsystem ("archive.pl $Noutfile 1>$outfile 2>&1 &");
	exit;
    }
    goodbye "re-did unfinished images";
}
    
print STDERR "no unfinished nights, slurping a new set\n";
if (-e $scriptfile) { if (vsystem ("rm $scriptfile")) { goodbye "couldn't delete old MAESTRO file"; }}

if (vsystem ("rsh -n pallas slurp.pl")) { goodbye "couldn't start slurp on pallas"; }
if (-e $killfile) { goodbye "process killed with $killfile"; }
if (!-e $scriptfile) {goodbye "slurp failed to download files"; }

print STDERR "done slurping, starting controller runs\n";
vsystem ("date");

# run controller on the resulting images
# controller wants to be run in the same directory as the inlists 
chdir $scriptdir;
open (MAESTRO, $scriptfile);
while ($line = <MAESTRO>) {
    @entries = split (" ",$line);
    $inlist = $entries[0] . ".inlist";
    $outlist = $entries[0] . ".outlist";
    $subdir = $entries[0];
    $photcode = $entries[1];
    docontroller ($subdir, $inlist, $outlist, $photcode, "-s flatten", 1);
    print STDERR "done: $subdir\n";
    open (DONELOG, ">>$donelogfile");
    print DONELOG "$entries[0]\n";
    close (DONELOG);
}
close (MAESTRO);

unless (unlink ($alockfile)) { goodbye "can't unlink $alockfile, why not?"; }
print STDERR "ending execution\n";
vsystem ("date");
print STDERR "finished new images";

unless (-e $haltfile) { 
    print STDERR "starting a new archive.pl with output to $outfile\n";
    vsystem ("notify.pl $ARGV[0] $outfile $Noutfile");
    vsystem ("archive.pl $Noutfile 1>$outfile 2>&1 &");
} else {
    vsystem ("notify.pl $ARGV[0] none 0");
}
exit;

# run nrphot on the altered cpt files and quit
chdir $database;
@files = `find . -name "*.cpt" -newer $reffile`;
$Nfiles = @files;
print STDERR "N: $Nfiles\n";
foreach $f (@files) {
    chop ($f);
    vsystem ("nrphot -v $f");
    if (-e $lockfile) { goodbye "lock file exists"; }
}

# delete inlist files

# do controller (& cleaning if needed), taking care of flats and error conditions
sub docontroller {
    if (-e $killfile) { goodbye "process killed with $killfile"; }
    if (-e $clockfile) { goodbye "lock file $clockfile exists"; }
    # break up entry line
    $outsubdir = $_[0];
    $in = $_[1];
    $out = $_[2];
    $phot = $_[3];
    $flags = $_[4];
    $clean = $_[5];
    # create sub-directory in output dir
    $outdir = $workspace . $outsubdir;
    unless (-e $outdir) { unless (mkdir ($outdir,0777)) { goodbye "cannot create output directory $outdir"; } }
    # make link to correct flat
    $flatlink = $flatdir . "newflat.fits";
    $realflat = $flatdir . "flatcode" . $phot . ".fits";
    if (-l $flatlink) { unless (unlink ($flatlink)) { goodbye "cannot unlink $flatlink"; }}
    if (-e $flatlink) { goodbye "$flatlink exists, but is not a link!"; }
    unless (symlink ($realflat, $flatlink)) { goodbye "cannot symlink $realflat to $flatlink"; }
    # run controller and cleanup
    if (-e $lockfile) { goodbye "lock file exists"; }
    vsystem ("controller $flags -o $outsubdir $in $out $phot");
    if (-e $clockfile) { goodbye "lock file $clockfile exists"; }
    if (-e $lockfile) { goodbye "lock file exists"; }
    if (-e $killfile) { goodbye "process killed with $killfile"; }
    if ($clean) {
	vsystem ("cleaning.pl");
	if (-e $lockfile) { goodbye "lock file exists"; }
    }
}

sub by_filename {
    @value1 = split (" ", $a);
    @value2 = split (" ", $b);
    $value1[0] cmp $value2[0]; 
}
    
# we read from in and write to out those files with 1st 0 in the mode column
sub parse_outlist {
    $in = $_[0];
    $out = $_[1];
    $mode = $_[2];
    
# load inlist
    open (INLIST, $in);
    @inarray = ();
    while ($line = <INLIST>) {
	chop ($line);
	@inarray = (@inarray,$line);
    }
    $Nin = @inarray;
    close (INLIST);
    @inarray = sort by_filename @inarray;

    @outarray = ();
    for ($k = 0; $k < $Nin; $k++) {
	@pvalues = split (" ", $inarray[$k]);
	$more = 1;
	while ($more) {
	    print STDERR ".";
	    @nvalues = split (" ", $inarray[$k+1]);
	    if ($nvalues[0] cmp $pvalues[0]) {
		$more = 0;
	    } else {
		$pvalues[1] |= $nvalues[1];
		$pvalues[2] |= $nvalues[2];
		$pvalues[3] |= $nvalues[3];
		$pvalues[4] |= $nvalues[4];
		$pvalues[5] |= $nvalues[5];
		$k ++;
	    }
	}
	# check the earlier status values: if failed earlier, skip
	$skip = 0;
	for ($m = 1; $m < $mode; $m++) { if (!$pvalues[$m]) { $skip = 1; } }
	if (! $skip && !$pvalues[$mode]) {
	    $name = $pvalues[0];
	    $name =~ s/$dumpspace//;
	    $name =~ s/://;
	    @outarray = (@outarray,$name);
	}
    }
    $Nout = @outarray;
    if ($Nout) {
	open (OUTLIST, ">$out");
	for ($k = 0; $k < $Nout; $k++) {
	    print OUTLIST "$outarray[$k]\n";
	}
	close (OUTLIST);
    }
    $Nout;
}

sub parse_inlist {
    $in = $_[0];
    $out = $_[1];
    $new = $_[2];
    
# load inlist
    open (INLIST, $in);
    @inarray = ();
    while ($line = <INLIST>) {
	chop ($line);
	@inarray = (@inarray,$line);
    }
    $Nin = @inarray;
    close (INLIST);

# load inlist
    open (OUTLIST, $out);
    @outarray = ();
    while ($line = <OUTLIST>) {
	chop ($line);
	@outarray = (@outarray,$line);
    }
    $Nout = @outarray;
    close (OUTLIST);

# load outlist
    @newarray = ();
    for ($k = 0; $k < $Nin; $k++) {
	$gotit = 0;
	for ($m = 0; ! $gotit && ($m < $Nout); $m++) {
	    if ($outarray[$m] =~ /$inarray[$k]/) {
		$gotit = 1;
	    }
	}
	if (!$gotit) {
	    @newarray = (@newarray,$inarray[$k]);
	}
    }
    $Nnew = @newarray;
    if ($Nnew) {
	open (NEWLIST, ">$new");
	for ($k = 0; $k < $Nnew; $k++) {
	    print NEWLIST "$inarray[$k]\n";
	}
	close (NEWLIST);
    }
    $Nnew;
}
