#!/usr/bin/env perl

# this script is used to cleanup the files from the different ipp
# stages.  It can be called with one of two cleanup modes: clean and
# purge.  the former removes temporary data files, leaving behind
# enough information for the results to be rebuilt.  The latter
# removes all but basic logging data.

use warnings;
use strict;
use Carp;

use IPC::Cmd 0.36 qw( can_run run );
use File::Spec;
use PS::IPP::Config 1.01 qw( :standard );
use PS::IPP::Metadata::List qw( parse_md_list );
use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
use Pod::Usage qw( pod2usage );

my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions

# Parse the command-line arguments
my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist);
GetOptions('stage=s'        => \$stage,     # which analysis stage to clean?
	   'camera|i=s'     => \$camera,    # user-supplied camera name
	   'stage_id=s'     => \$stage_id,  # id for this stage (only needed for certain stages)
	   'mode|m=s'       => \$mode,      # cleanup mode (clean / purge)
	   'path_base=s'    => \$path_base, # basename for files
	   'dbname|d=s'     => \$dbname,    # Database name
           'verbose'        => \$verbose,   # Print to stdout
	   'no-op'          => \$no_op,     # pretend but don't actually inject
	   'helplist'       => \$helplist   # give help listing
	   ) or pod2usage( 2 );

pod2usage( -msg => "remove temporary / all data files for an IPP analysis stage", 
	   -exitval => 2) if defined $helplist;

pod2usage( -msg => "Usage: $0 --camera (name) --stage (stage) --stage_id (stage_id) --mode (mode) [--path_base (path)] [--dbname dbname] [--no-op] [--help]", 
	   -exitval => 2 ) if scalar @ARGV;

pod2usage( -msg => "Required options:--camera (name) --stage (stage) --mode (mode)",
	   -exitval => 3) unless 
    defined $camera and
    defined $stage and
    defined $mode;

# $mode must be one of "goto_cleaned", "goto_scrubbed", or "goto_purged"
# goto_cleaned and goto_scrubbed both result in 'cleaned': scrubbed allows chips without config files to 
# be cleaned (they cannot be recovered, but the small data is left behind)
unless (($mode eq "goto_cleaned") || ($mode eq "goto_scrubbed") || ($mode eq "goto_purged")) {
    die "invalid cleanup mode $mode\n";    
}

my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff  => 1);
unless ($stages{$stage}) {
    die "unknown stage $stage for ipp_cleanup.pl\n";
}

$ipprc->define_camera($camera);

my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files

# choice of files to delete depends on the stage
if ($stage eq "chip") {
    
    die "--stage_id required for stage chip\n" if !$stage_id;
    ### select the imfiles for this entry

    # this stage uses 'chiptool'
    my $chiptool = can_run('chiptool') or die "Can't find chiptool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $imfiles;                      # Array of component files
    my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    }
    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);

    # extract the metadata for the files into a hash list
    $imfiles = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "chip", $stage_id, $PS_EXIT_PROG_ERROR);

    # loop over all of the imfiles, determine the path_base and class_id for each
    foreach my $imfile (@$imfiles) {
	my $class_id = $imfile->{class_id};
	my $path_base = $imfile->{path_base};
        my $status = 1;

        # don't clean up unless the data needed to update is available
	# modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
        if ($mode eq "goto_cleaned") {
            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);

            if (!$config_file or ! -e $config_file) {
                print STDERR "skipping cleanup for chipRun $stage_id $class_id "
                    . " because config file is missing\n";
                $status = 0;
            }
        }

        if ($status) {
            # array of actual filenames to delete
            my @files = ();

            # delete the temporary image datafiles
            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
            if ($mode eq "goto_purged") {
                # additional files to remove for 'purge' mode
                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.JPEG", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
            }
	
            # actual command to delete the files
            $status = &delete_files (\@files);
        }

	if ($status)  {
	    my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
            if ($mode eq "goto_purged") {
                $command .= " -topurgedimfile";
            } elsif ($mode eq "goto_scrubbed") {
                $command .= " -tocleanedimfile_from_scrubbed";
	    } else {
                $command .= " -tocleanedimfile";
            }
	    $command .= " -dbname $dbname" if defined $dbname;

	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
	    }
        } else {
	    # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will stop be run
	    my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -code 1";
	    $command .= " -dbname $dbname" if defined $dbname;

	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
	    }
	}
    }
    exit 0;
} 

if ($stage eq "camera") {
    die "--stage_id required for stage camera\n" if !$stage_id;
    # this stage uses 'camtool'
    my $camtool = can_run('camtool') or die "Can't find camtool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $exps;                      # Array of component files
    my $command = "$camtool -pendingcleanupexp -cam_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code);
    }
    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "camera", $stage_id, $PS_EXIT_PROG_ERROR);

    $exps = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "camera", $stage_id, $PS_EXIT_PROG_ERROR);

    my $n_exps = @$exps;
    &my_die("unexpected number of exposures $n_exps", "camera", $stage_id, $PS_EXIT_PROG_ERROR)
        if $n_exps != 1;

    my $exp = $exps->[0];
    my $path_base = $exp->{path_base};

    my $status = 1;
    # don't clean up unless the data needed to update is available
    if ($mode eq "goto_cleaned") {
        my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);

        if (!$config_file or ! -e $config_file) {
            print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n";
            $status = 0;
        }
    }
    if ($status) {
        my @files = ();
        # delete the temporary image datafiles
        addFilename (\@files, "PSASTRO.OUTPUT", $path_base);
        if ($mode eq "goto_purged") {
            # additional files to remove for 'purge' mode
            addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
            addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
            addFilename (\@files, "PSASTRO.STATS", $path_base);
        }
        # actual command to delete the files
        $status = &delete_files (\@files);
    }

    if ($status)  {
        my $command = "$camtool -cam_id $stage_id -updaterun";
        if ($mode eq "goto_cleaned") {
            $command .= " -state cleaned";
        } else {
            $command .= " -state purged";
        }
        $command .= " -dbname $dbname" if defined $dbname;
        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
            run(command => $command, verbose => $verbose);
        unless ($success) {
            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
            &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code);
        }
    } else {
        my $command = "$camtool -updateprocessedexp -cam_id $stage_id -code 1";
        $command .= " -dbname $dbname" if defined $dbname;

        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
            run(command => $command, verbose => $verbose);
        unless ($success) {
            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
            &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code);
        }
        exit $PS_EXIT_UNKNOWN_ERROR;
    }
    exit 0;
} 

if ($stage eq "warp") {
    die "--stage_id required for stage warp\n" if !$stage_id;
    # this stage uses 'warptool'
    my $warptool = can_run('warptool') or die "Can't find warptool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $skyfiles;                      # Array of component files
    my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
            run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
    }
    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR);

    $skyfiles = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "warp", $stage_id, $PS_EXIT_PROG_ERROR);

    my @files = ();
    foreach my $skyfile (@$skyfiles) {
        my $path_base = $skyfile->{path_base};
        my $skycell_id = $skyfile->{skycell_id};

        my $status = 1;
        if ($mode eq "goto_cleaned") {
            my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);

            if (!$config_file or ! -e $config_file) {
                print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
                    " because config file is missing\n";
                $status = 0;
            }
        }
        if ($status) {
            # delete the temporary image datafiles
            addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id );
            addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
            addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id);
            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);

            if ($mode eq "goto_purged") {
                # additional files to remove for 'purge' mode
                addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id );
                addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id );
                addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id );
                # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base);

                # XXX: do we want to delete these?
                # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id);
                # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
            }
            # actual command to delete the files
            $status = &delete_files (\@files);
        }

	if ($status)  {
	    my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
            if ($mode eq "goto_purged") {
                $command .= " -topurgedskyfile";
            } else {
                $command .= " -tocleanedskyfile";
            }
	    $command .= " -dbname $dbname" if defined $dbname;

	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
	    }
         } else {
            # XXX: -updateskyfile mode does not exist, need to add it
	    my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -code 1";
	    $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
	    }
            exit $PS_EXIT_UNKNOWN_ERROR;
	}
    }
    exit 0;
}

# left TODO
# fake : faketool : -pendingcleanupimfile (loop over imfiles)
# stack: stacktool : -pendingcleanupskyfile (loop over skyfiles)
# diff:  difftool : -pendingcleanupskyfile

die "ipp_cleanup.pl -stage $stage not yet implemented\n";

sub delete_files 
{
    my $files = shift; # reference to a list of files to unlink
    
    # this script is, of course, very dangerous.  
    foreach my $file (@$files) {
	print STDERR "unlinking $file\n";
        $ipprc->file_delete($file);
    }
    return 1;
}

sub addFilename 
{
    my $files      = shift; # reference to a list of files to unlink
    my $filerule   = shift; # filerule to add
    my $path_base  = shift; # base filename
    my $class_id   = shift; # class_id, if needed

    my $file = $ipprc->filename($filerule, $path_base, $class_id);

    push @$files, $file;
    return 1;
}

sub my_die
{
    my $msg = shift; # Warning message on die
    my $stage = shift; # stage name
    my $stage_id = shift; #  identifier
    my $exit_code = shift; # Exit code
    # outputImage and path_base are globals

    carp($msg);
    exit $exit_code;
}

__END__
