#!/usr/bin/env perl

# in this script, we run flatten, dophot, fstat, and gastro
# these can be run totally in parallel, so this routine runs
# independently of the series processing (addstar, markstar, etc)
# in stage2

# for now these directories are hard-wired.  eventually, they
# should be looked up in the config file, so all scripts are
# consistent
$dumpspace = "/pallas/d1/";
$workspace = "/metis/d27/workspace/";
# $workspace = "/irene/d11/workspace/";

$scriptdir = "/hebe/d27/logs/";
$donelogfile = $scriptdir . "processed.log";
$scriptfile  = $scriptdir . "extracted.log";
$flatdir = "/hebe/d27/references/config/";

$database = "/hebe/d27/database/";
$alockfile = $database . "alock1";     # lock on stage1 (only 1 running at a time)
$clockfile = $database . "clock1";     # lock on controller
$killfile  = $database . "kill";       # quickly stop the processing
$haltfile  = $database . "halt";       # halt processing after this loop
$reffile = $database . "start.clean";

sub vsystem {
    print STDERR "@_\n";
    $status = system ("@_");
    $status;
}

sub goodbye {
    print STDERR "ending execution\n";
    vsystem ("date");
    vsystem ("notify.pl $ARGV[0] none 0");
    die "@_";
}

if (@ARGV < 1) {die "USAGE: stage1 Noutfile\n"; }
$Noutfile = $ARGV[0] + 1;
$outfile = "stage1." . $Noutfile . ".log";

print STDERR "starting execution\n";
vsystem ("date");
if (-e $killfile) { goodbye "process killed with $killfile"; }


if (-e $alockfile) { goodbye "archive lock $alockfile is set"; }
open (TEMP, ">$alockfile");
close (TEMP);

# load in done date list
open (DONELOG, $donelogfile);
@donelist = ();
while ($line = <DONELOG>) {
  chop ($line);
  @donelist = (@donelist,$line);
}
$Ndone = @donelist;
close (DONELOG);

# load in maestro list
open (MAESTRO, $scriptfile);
@todolist = ();
while ($line = <MAESTRO>) {
  chop ($line);
  @todolist = (@todolist,$line);
}
$Ntodo = @todolist;
close (MAESTRO);

# find entries in MAESTRO that are NOT in DONELOG
@leftlist = ();
for ($i = 0; $i < $Ntodo; $i ++) {
    $gotit = 0;
    for ($j = 0; ($j < $Ndone) && ! $gotit; $j ++) {
	@todo = split (" ",$todolist[$i]);
	@done = split (" ",$donelist[$j]);
	if ($todo[0] eq $done[0]) {
	    $gotit = 1;
	}
    }
    if (!$gotit) {
	@leftlist = (@leftlist, $todolist[$i]);  
    }
}
$Nleft = @leftlist;

if (-e $killfile) { goodbye "process killed with $killfile"; }
# run controller on unfinished entries
# we are going to run this in the script directory
if ($Nleft) {
    chdir $scriptdir;
    for ($i = 0; $i < $Nleft; $i++) {
	print STDERR "unfinished: $leftlist[$i]\n";
	@entries = split (" ",$leftlist[$i]);
	$inlist = $entries[0] . ".inlist";
	$outlist = $entries[0] . ".addlist";
	$subdir = $entries[0];
	$photcode = $entries[1];
	print STDERR "$subdir: $inlist $outlist $photcode\n";
	if (-e $outlist) {
	    # partially successful run, restart controller in the middle
	    $tmp_inlist = $entries[0] . ".tmp_inlist";
	    $tmpoutlist = $entries[0] . ".tmpaddlist";
	    if (-e $tmpoutlist) { if (vsystem ("cat $tmpoutlist >> $outlist")) { goodbye "trouble merging $tmpoutlist and $outlist\n"; }}
	    # partially successful re-start: make sure $outlist is complete
	    if (parse_outlist ($outlist,$tmp_inlist,1)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s flatten", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    if (parse_outlist ($outlist,$tmp_inlist,2)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s dophot", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    if (parse_outlist ($outlist,$tmp_inlist,3)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s fstat", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    if (parse_outlist ($outlist,$tmp_inlist,4)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s gastro", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    # do inlist entries not in outlist
	    if (parse_inlist ($inlist,$outlist,$tmp_inlist)) {
		docontroller ($subdir, $tmp_inlist, $tmpoutlist, $photcode, "-s flatten", 0);
		vsystem ("cat $tmpoutlist >> $outlist");
	    }
	    unless (unlink ($tmpoutlist)) { goodbye "can't unlink $tmpoutlist, why not?"; }
	    unless (unlink ($tmp_inlist)) { goodbye "can't unlink $tmp_inlist, why not?"; }
	    # done with this night, add to donelogfile
	    print STDERR "done: $subdir $photcode\n";
	    open (DONELOG, ">>$donelogfile");
	    print DONELOG "$subdir $photcode\n";
	    close (DONELOG);
	} else {
	    docontroller ($subdir, $inlist, $outlist, $photcode, "-s flatten", 1);
	    print STDERR "done: $subdir $photcode\n";
	    open (DONELOG, ">>$donelogfile");
	    print DONELOG "$subdir $photcode\n";
	    close (DONELOG);
	}
    }
    unless (unlink ($alockfile)) { goodbye "can't unlink $alockfile, why not?"; }
    print STDERR "remember to delete images from complete directories\n";

    unless (-e $haltfile) { 
	print STDERR "re-did unfinished images\n";
	print STDERR "starting a new stage1 with output to $outfile\n";
	vsystem ("notify.pl $ARGV[0] $outfile $Noutfile");
	vsystem ("stage1 $Noutfile 1>$outfile 2>&1 &");
	exit;
    }
    goodbye "re-did unfinished images";
}
    
# get a new set of data
print STDERR "no unfinished nights, slurping a new set\n";
if (-e $scriptfile) { if (vsystem ("rm $scriptfile")) { goodbye "couldn't delete old MAESTRO file"; }}

if (vsystem ("rsh -n pallas slurp")) { goodbye "couldn't start slurp on pallas"; }
if (-e $killfile) { goodbye "process killed with $killfile"; }
if (!-e $scriptfile) {goodbye "slurp failed to download files"; }

print STDERR "done slurping, starting controller runs\n";
vsystem ("date");

# run controller on the resulting images
# controller wants to be run in the same directory as the inlists 
$Nruns = 0;
chdir $scriptdir;
open (MAESTRO, $scriptfile);
while ($line = <MAESTRO>) {
    @entries = split (" ",$line);
    $inlist = $entries[0] . ".inlist";
    $outlist = $entries[0] . ".addlist";
    $subdir = $entries[0];
    $photcode = $entries[1];
    docontroller ($subdir, $inlist, $outlist, $photcode, "-s flatten", 1);
    print STDERR "done: $subdir $photcode\n";
    open (DONELOG, ">>$donelogfile");
    print DONELOG "$subdir $photcode\n";
    close (DONELOG);
    $Nruns ++;
}
close (MAESTRO);

if (Nruns == 0) {
    print STDERR "no data downloaded in slurp, waiting for a while\n";
    vsystem ("sleep 3600");
}
unless (unlink ($alockfile)) { goodbye "can't unlink $alockfile, why not?"; }
print STDERR "ending execution\n";
vsystem ("date");
print STDERR "finished new images";

unless (-e $haltfile) { 
    print STDERR "starting a new stage1 with output to $outfile\n";
    vsystem ("notify.pl $ARGV[0] $outfile $Noutfile");
    vsystem ("stage1 $Noutfile 1>$outfile 2>&1 &");
} else {
    vsystem ("notify.pl $ARGV[0] none 0");
}
exit;

# do controller (& cleaning if needed), taking care of flats and error conditions
sub docontroller {
    if (-e $killfile) { goodbye "process killed with $killfile"; }
    if (-e $clockfile) { goodbye "lock file $clockfile exists"; }
    # break up entry line
    $outsubdir = $_[0];
    $in = $_[1];
    $out = $_[2];
    $phot = $_[3];
    $flags = $_[4];
    $clean = $_[5];
    # create sub-directory in output dir
    $outdir = $workspace . $outsubdir;
    unless (-e $outdir) { unless (mkdir ($outdir,0777)) { goodbye "cannot create output directory $outdir"; } }
    # make link to correct flat
    $flatlink = $flatdir . "newflat.fits";
    $realflat = $flatdir . "flatcode" . $phot . ".fits";
    if (-l $flatlink) { unless (unlink ($flatlink)) { goodbye "cannot unlink $flatlink"; }}
    if (-e $flatlink) { goodbye "$flatlink exists, but is not a link!"; }
    unless (symlink ($realflat, $flatlink)) { goodbye "cannot symlink $realflat to $flatlink"; }
    # run controller and cleanup
    vsystem ("control1 $flags -o $outsubdir $in $out $phot");
    if (-e $clockfile) { goodbye "lock file $clockfile exists"; }
    if (-e $killfile) { goodbye "process killed with $killfile"; }
}

sub by_filename {
    @value1 = split (" ", $a);
    @value2 = split (" ", $b);
    $value1[0] cmp $value2[0]; 
}
    
# we read from in and write to out those files with 1st 0 in the mode column
sub parse_outlist {
    $in = $_[0];
    $out = $_[1];
    $mode = $_[2];
    
# load inlist
    open (INLIST, $in);
    @inarray = ();
    while ($line = <INLIST>) {
	chop ($line);
	@inarray = (@inarray,$line);
    }
    $Nin = @inarray;
    close (INLIST);
    @inarray = sort by_filename @inarray;

    @outarray = ();
    for ($k = 0; $k < $Nin; $k++) {
	@pvalues = split (" ", $inarray[$k]);
	$more = 1;
	while ($more) {
	    print STDERR ".";
	    @nvalues = split (" ", $inarray[$k+1]);
	    if ($nvalues[0] cmp $pvalues[0]) {
		$more = 0;
	    } else {
		$pvalues[1] |= $nvalues[1];
		$pvalues[2] |= $nvalues[2];
		$pvalues[3] |= $nvalues[3];
		$pvalues[4] |= $nvalues[4];
		$pvalues[5] |= $nvalues[5];
		$k ++;
	    }
	}
	# check the earlier status values: if failed earlier, skip
	$skip = 0;
	for ($m = 1; $m < $mode; $m++) { if (!$pvalues[$m]) { $skip = 1; } }
	if (! $skip && !$pvalues[$mode]) {
	    $name = $pvalues[0];
	    $name =~ s/$dumpspace//;
	    $name =~ s/://;
	    @outarray = (@outarray,$name);
	}
    }
    $Nout = @outarray;
    if ($Nout) {
	open (OUTLIST, ">$out");
	for ($k = 0; $k < $Nout; $k++) {
	    print OUTLIST "$outarray[$k]\n";
	}
	close (OUTLIST);
    }
    $Nout;
}

sub parse_inlist {
    $in = $_[0];
    $out = $_[1];
    $new = $_[2];
    
# load inlist
    open (INLIST, $in);
    @inarray = ();
    while ($line = <INLIST>) {
	chop ($line);
	@inarray = (@inarray,$line);
    }
    $Nin = @inarray;
    close (INLIST);

# load inlist
    open (OUTLIST, $out);
    @outarray = ();
    while ($line = <OUTLIST>) {
	chop ($line);
	@outarray = (@outarray,$line);
    }
    $Nout = @outarray;
    close (OUTLIST);

# load outlist
    @newarray = ();
    for ($k = 0; $k < $Nin; $k++) {
	$gotit = 0;
	for ($m = 0; ! $gotit && ($m < $Nout); $m++) {
	    if ($outarray[$m] =~ /$inarray[$k]/) {
		$gotit = 1;
	    }
	}
	if (!$gotit) {
	    @newarray = (@newarray,$inarray[$k]);
	}
    }
    $Nnew = @newarray;
    if ($Nnew) {
	open (NEWLIST, ">$new");
	for ($k = 0; $k < $Nnew; $k++) {
	    print NEWLIST "$newarray[$k]\n";
	}
	close (NEWLIST);
    }
    $Nnew;
}
