#!/usr/bin/env perl

# this script is used to cleanup the files from the different ipp
# stages.  It can be called with one of two cleanup modes: clean and
# purge.  the former removes temporary data files, leaving behind
# enough information for the results to be rebuilt.  The latter
# removes all but basic logging data.

use warnings;
use strict;
use Carp;

use IPC::Cmd 0.36 qw( can_run run );
use File::Spec;
use PS::IPP::Config 1.01 qw( :standard );
use PS::IPP::Metadata::List qw( parse_md_list );
use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
use Pod::Usage qw( pod2usage );

# Parse the command-line arguments
my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist);
GetOptions('stage=s'        => \$stage,     # which analysis stage to clean?
           'camera|i=s'     => \$camera,    # user-supplied camera name
           'stage_id=s'     => \$stage_id,  # id for this stage (only needed for certain stages)
           'mode|m=s'       => \$mode,      # cleanup mode (clean / purge)
           'path_base=s'    => \$path_base, # basename for files
           'dbname|d=s'     => \$dbname,    # Database name
           'verbose'        => \$verbose,   # Print to stdout
           'no-op'          => \$no_op,     # pretend but don't actually inject
           'helplist'       => \$helplist   # give help listing
           ) or pod2usage( 2 );

pod2usage( -msg => "remove temporary / all data files for an IPP analysis stage",
           -exitval => 2) if defined $helplist;

pod2usage( -msg => "Usage: $0 --camera (name) --stage (stage) --stage_id (stage_id) --mode (mode) [--path_base (path)] [--dbname dbname] [--no-op] [--help]",
           -exitval => 2 ) if scalar @ARGV;

pod2usage( -msg => "Required options:--camera (name) --stage (stage) --mode (mode)",
           -exitval => 3) unless
    defined $camera and
    defined $stage and
    defined $mode;

my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up", $stage_id, $PS_EXIT_CONFIG_ERROR); # this is used for PATH, NEB filename conversions

# $mode must be one of "goto_cleaned", "goto_scrubbed", or
# "goto_purged" goto_cleaned and goto_scrubbed both result in
# 'cleaned' on success ('scrubbed' allows chips without config files
# to be cleaned; they cannot be recovered, but the small data is left
# behind). XXX make 'scrubbed' a data_state?
unless (($mode eq "goto_cleaned") || ($mode eq "goto_scrubbed") || ($mode eq "goto_purged")) {
    die "invalid cleanup mode $mode\n";
}

my $error_state;
if ($mode eq "goto_cleaned")  { $error_state = "error_cleaned";  }
if ($mode eq "goto_scrubbed") { $error_state = "error_scrubbed"; }
if ($mode eq "goto_purged")   { $error_state = "error_purged";   }


my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff"  => 1,
	       "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1,
	       "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1,
	       "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 );
unless ($stages{$stage}) {
    die "unknown stage $stage for ipp_cleanup.pl\n";
}

my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files

# choice of files to delete depends on the stage
if ($stage eq "chip") {

    die "--stage_id required for stage chip\n" if !$stage_id;
    ### select the imfiles for this entry

    # this stage uses 'chiptool'
    my $chiptool = can_run('chiptool') or die "Can't find chiptool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $imfiles;                      # Array of component files
    my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    }

    # if there are no chipProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
    if (@$stdout_buf == 0)  {
	my $command = "$chiptool -chip_id $stage_id -updaterun -set_state new";
	$command .= " -dbname $dbname" if defined $dbname;

	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
	    run(command => $command, verbose => $verbose);
	unless ($success) {
	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	    &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
	}
	exit 0;
    }

    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);

    # extract the metadata for the files into a hash list
    $imfiles = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "chip", $stage_id, $PS_EXIT_PROG_ERROR);

    # loop over all of the imfiles, determine the path_base and class_id for each
    foreach my $imfile (@$imfiles) {
        my $class_id = $imfile->{class_id};
        my $path_base = $imfile->{path_base};
        my $status = 1;

        # don't clean up unless the data needed to update is available
        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
	# goto_scrubbed now requires the config file to not exist.
        if ($mode eq "goto_cleaned") {
            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);

            if (!$config_file or ! -e $config_file) {
                print STDERR "skipping cleanup for chipRun $stage_id $class_id "
                    . " because config file is missing\n";
                $status = 0;
            }
        }
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);

	    if ($config_file and -e $config_file) {
		print STDERR "skipping scrubbed for chipRun $stage_id $class_id "
		    . " because config file is present\n";
		$status = 0;
	    }
	}

        if ($status) {
            # array of actual filenames to delete
            my @files = ();

            # delete the temporary image datafiles
            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
            if ($mode eq "goto_purged") {
                # additional files to remove for 'purge' mode
                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);          # clean?
                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);          # clean?
                addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);         #clean?
                addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
            }

            # actual command to delete the files
            $status = &delete_files (\@files);
        }

        if ($status)  {
            my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
            if ($mode eq "goto_purged") {
                $command .= " -topurgedimfile";
            }
	    elsif ($mode eq "goto_cleaned") {
                $command .= " -tocleanedimfile";
            }
	    elsif ($mode eq "goto_scrubbed") {
		$command .= " -toscrubbedimfile";
	    }

            $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
            }
        } else {

	    # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
	    my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -set_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
            }
        }
    }
    exit 0;
}

if ($stage eq "camera") {
    die "--stage_id required for stage camera\n" if !$stage_id;
    # this stage uses 'camtool'
    my $camtool = can_run('camtool') or die "Can't find camtool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $exps;                      # Array of component files
    my $command = "$camtool -pendingcleanupexp -cam_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code);
    }
    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "camera", $stage_id, $PS_EXIT_PROG_ERROR);

    $exps = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "camera", $stage_id, $PS_EXIT_PROG_ERROR);

    my $n_exps = @$exps;
    &my_die("unexpected number of exposures $n_exps", "camera", $stage_id, $PS_EXIT_PROG_ERROR)
        if $n_exps != 1;

    my $exp = $exps->[0];
    my $path_base = $exp->{path_base};

    my $status = 1;
    # don't clean up unless the data needed to update is available
    # goto_scrubbed now requires the config file to not be present
    if ($mode eq "goto_cleaned") {
        my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);

        if (!$config_file or ! -e $config_file) {
            print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n";
            $status = 0;
        }
    }
    elsif ($mode eq "goto_scrubbed") {
	my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);

	if ($config_file and -e $config_file) {
	    print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n";
	    $status = 0;
	}
    }
    if ($status) {
        my @files = ();
        # delete the temporary image datafiles
#        addFilename (\@files, "PSASTRO.OUTPUT", $path_base);
        if ($mode eq "goto_purged") {
            # additional files to remove for 'purge' mode
            addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
            addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
            addFilename (\@files, "PSASTRO.STATS", $path_base);
        }
        # actual command to delete the files
        $status = &delete_files (\@files);
    }

    if ($status)  {
        my $command;
        if ($mode eq "goto_cleaned") {
            $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";
	}
        if ($mode eq "goto_scrubbed") {
            $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed";
	}
        if ($mode eq "goto_purged") {
            $command = "$camtool -updaterun -cam_id $stage_id -set_state purged";
        }
        $command .= " -dbname $dbname" if defined $dbname;
        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
            run(command => $command, verbose => $verbose);
        unless ($success) {
            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
            &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code);
        }
    } else {
	# since 'camera' has only a single imfile, we can just update the run
        my $command = "$camtool -updaterun -cam_id $stage_id -set_state $error_state";
        $command .= " -dbname $dbname" if defined $dbname;

        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
            run(command => $command, verbose => $verbose);
        unless ($success) {
            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
            &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code);
        }
        exit $PS_EXIT_UNKNOWN_ERROR;
    }
    exit 0;
}

if ($stage eq "warp") {
    die "--stage_id required for stage warp\n" if !$stage_id;
    # this stage uses 'warptool'
    my $warptool = can_run('warptool') or die "Can't find warptool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $skyfiles;                      # Array of component files
    my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
            run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
    }
    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR);

    $skyfiles = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "warp", $stage_id, $PS_EXIT_PROG_ERROR);

    my @files = ();
    foreach my $skyfile (@$skyfiles) {
        my $path_base = $skyfile->{path_base};
        my $skycell_id = $skyfile->{skycell_id};

        my $status = 1;
        if ($mode eq "goto_cleaned") {
            my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);

            if (!$config_file or ! -e $config_file) {
                print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
                    " because config file is missing\n";
                $status = 0;
            }
        }
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);

	    if ($config_file and -e $config_file) {
		print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" .
		    " because config file is present\n";
		$status = 0;
	    }
	}
        if ($status) {
            # delete the temporary image datafiles
            addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id );
            addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
            addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id);
#            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);

            if ($mode eq "goto_purged") {
                # additional files to remove for 'purge' mode
                addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id );
                addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id );
                addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id );
                # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base);

                # XXX: do we want to delete these?
                # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id);
                # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
            }
            # actual command to delete the files
            $status = &delete_files (\@files);
        }

        if ($status)  {
            my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
            if ($mode eq "goto_purged") {
                $command .= " -topurgedskyfile";
            } 
	    elsif ($mode eq "goto_cleaned") {
                $command .= " -tocleanedskyfile";
            }
	    elsif ($mode eq "goto_scrubbed") {
		$command .= " -toscrubbedskyfile";
	    }
            $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
            }
         } else {
	    my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                run(command => $command, verbose => $verbose);
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
            }
            exit $PS_EXIT_UNKNOWN_ERROR;
        }
    }
    exit 0;
}

### added for cleanup, based on warp stage entry
if ($stage eq 'stack') {
    
    die "--stage_id required for stage stack\n" if !$stage_id;

    # this stage uses 'stacktool'
    my $stacktool = can_run('stacktool') or die "Can't find stacktool";

    # Get list of component imfiles
    my $skyfiles;                  # Array reference of component files
    my $command = "stacktool -pendingcleanupskyfile -stack_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
    }
    my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 
	&my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR);

    $skyfiles = parse_md_list($metadata) or 
	&my_die("Unable to parse metadata list", "stack", $stage_id, $PS_EXIT_PROG_ERROR);
    
    my @files = ();
    foreach my $skyfile (@{ $skyfiles }) {
	my $path_base = $skyfile->{path_base};
	my $skycell_id = $skyfile->{skycell_id};

	my $status = 1;
	if ($mode eq "goto_cleaned") {
	    my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);

	    $config_file =~ s%^file://%%;
	    if (!$config_file or ! -e $config_file) {
		print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" .
		    " because config file is missing\n";
		$status = 0;
	    }
	    $config_file = 'file://' . $config_file;
	}
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);
	    $config_file =~ s%^file://%%;
	    if ($config_file and -e $config_file) {
		print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" .
		    " because config file is present\n";
		$status = 0;
	    }
	}
	if ($status) {
	    # delete the temporary image datafiles
	    addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id);
	    addFilename(\@files, "PPSTACK.OUTPUT.MASK", $path_base, $skycell_id);
	    addFilename(\@files, "PPSTACK.OUTPUT.VARIANCE", $path_base, $skycell_id);

	    if ($mode eq "goto_purged") {
		# additional files to remove for 'purge' mode
		addFilename(\@files, "PPSTACK.CONV.KERNEL", $path_base, $skycell_id);
		addFilename(\@files, "PPSTACK.OUTPUT.JPEG1", $path_base, $skycell_id);
		addFilename(\@files, "PPSTACK.OUTPUT.JPEG2", $path_base, $skycell_id);
		# Commented out to match warp files.
		#addFilename(\@files, "PPSTACK.TARGET.PSF", $path_base, $skycell_id);
		#addFilename(\@files, "PPSTACK.CONFIG", $path_base, $skycell_id);
	    }

	    $status = &delete_files(\@files);
	}

	if ($status) {
	    my $command = "$stacktool -stack_id $stage_id";
	    if ($mode eq "goto_purged") {
		$command .= " -updaterun -state purged";
	    } 
	    elsif ($mode eq "goto_cleaned") {
		$command .= " -updaterun -state cleaned";
	    }
	    elsif ($mode eq "goto_scrubbed") {
		$command .= " -updaterun -state scrubbed";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
	    }
	} else {
	    my $command = "$stacktool -updaterun  -stack_id $stage_id -state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
	    }
	    exit $PS_EXIT_UNKNOWN_ERROR;
	}
    }
    exit 0;
}

if ($stage eq 'diff') {
    
    die "--stage_id required for stage diff\n" if !$stage_id;

    # this stage uses 'difftool'
    my $difftool = can_run('difftool') or die "Can't find difftool";

    # Get list of component imfiles
    my $skyfiles;                  # Array reference of component files
    my $command = "difftool -pendingcleanupskyfile -diff_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
    }
    my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 
	&my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR);

    $skyfiles = parse_md_list($metadata) or 
	&my_die("Unable to parse metadata list", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
    
    my @files = ();
    foreach my $skyfile (@{ $skyfiles }) {
	my $path_base = $skyfile->{path_base};
	my $skycell_id = $skyfile->{skycell_id};

	my $status = 1;
	if ($mode eq "goto_cleaned") {
	    my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);

	    $config_file =~ s%^file://%%;
	    if (!$config_file or ! -e $config_file) {
		print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" .
		    " because config file ($config_file) is missing\n";
		$status = 0;
	    }
	    $config_file = 'file://' . $config_file;
	}
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
	    $config_file =~ s%^file://%%;
	    if ($config_file and -e $config_file) {
		print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" .
		    " because config file ($config_file) is present\n";
		$status = 0;
	    }
	}
	if ($status) {
	    # delete the temporary image datafiles
	    addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id);
	    addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id);
	    addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id);

	    addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id);
	    addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id);
	    addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id);

	    addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id);
	    addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id);
	    addFilename(\@files, "PPSUB.INPUT.CONV.VARIANCE", $path_base, $skycell_id);
	    
	    addFilename(\@files, "PPSUB.REF.CONV", $path_base, $skycell_id);
	    addFilename(\@files, "PPSUB.REF.CONV.MASK", $path_base, $skycell_id);
	    addFilename(\@files, "PPSUB.REF.CONV.VARIANCE", $path_base, $skycell_id);
	    
	    if ($mode eq "goto_purged") {
		# additional files to remove for 'purge' mode
		addFilename(\@files, "PPSUB.OUTPUT.KERNELS", $path_base, $skycell_id);
		addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id);
		addFilename(\@files, "PPSUB.OUTPUT.JPEG2", $path_base, $skycell_id);
		# Commented out to match warp files.
		#addFilename(\@files, "PPSUB.CONFIG", $path_base, $skycell_id);
		addFilename(\@files, "PPSUB.OUTPUT.SOURCES", $path_base, $skycell_id);
		addFilename(\@files, "PPSUB.INVERSE.SOURCES", $path_base, $skycell_id);
		
	    }
	    print STDERR "MY FILES: @files\n";
	    $status = &delete_files(\@files);
	}
	print STDERR "MY STATUS: $status\n";
	if ($status) {
	    my $command = "$difftool -diff_id $stage_id";

	    if ($mode eq "goto_purged") {
		$command .= " -updaterun -state purged";
	    }
	    elsif ($mode eq "goto_cleaned") {
		$command .= " -updaterun -state cleaned";
	    }
	    elsif ($mode eq "goto_scrubbed") {
		$command .= " -updaterun -state scrubbed";
	    }

	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
	    }
	} else {
	    my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state";

	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
	    }
	    exit $PS_EXIT_UNKNOWN_ERROR;
	}
    }
    exit 0;
}
if ($stage eq 'fake') {
    print STDERR "This does not seem to work at present, as no files exist. Terminating quietly.\n";
    exit(0);
    die "--stage_id required for stage fake\n" if !$stage_id;
    ### select the imfiles for this entry

    # this stage uses 'chiptool'
    my $faketool = can_run('faketool') or die "Can't find faketool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $imfiles;                      # Array of component files
    my $command = "$faketool -pendingcleanupimfile -fake_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
    }

    # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
    if (@$stdout_buf == 0)  {
	my $command = "$faketool -fake_id $stage_id -updaterun -set_state new";
	$command .= " -dbname $dbname" if defined $dbname;

	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
	    run(command => $command, verbose => $verbose);
	unless ($success) {
	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	    &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
	}
	exit 0;
    }

    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR);

    # extract the metadata for the files into a hash list
    $imfiles = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR);

    # loop over all of the imfiles, determine the path_base and class_id for each
    foreach my $imfile (@$imfiles) {
        my $class_id = $imfile->{class_id};
        my $path_base = $imfile->{path_base};
        my $status = 1;

        # don't clean up unless the data needed to update is available
        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
	# goto_scrubbed now requires the config file to not exist.
        if ($mode eq "goto_cleaned") {
            my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id);

            if (!$config_file or ! -e $config_file) {
                print STDERR "skipping cleanup for fakeRun $stage_id $class_id "
                    . " because config file is missing\n";
                $status = 0;
            }
        }
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id);

	    if ($config_file and -e $config_file) {
		print STDERR "skipping scrubbed for fakeRun $stage_id $class_id "
		    . " because config file is present\n";
		$status = 0;
	    }
	}

        if ($status) {
            # array of actual filenames to delete
            my @files = ();

            # delete the temporary image datafiles
            addFilename (\@files, "PPSIM.OUTPUT.MEF", $path_base, $class_id);
            addFilename (\@files, "PPSIM.OUTPUT.SPL", $path_base, $class_id);
            addFilename (\@files, "PPSIM.FAKE.CHIP", $path_base, $class_id);
            addFilename (\@files, "PPSIM.FORCE.CHIP", $path_base, $class_id);
            if ($mode eq "goto_purged") {
                # additional files to remove for 'purge' mode
		addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id);
		addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id);
		addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id);
            }

            # actual command to delete the files
            $status = &delete_files (\@files);
        }

        if ($status)  {
            my $command = "$faketool -fake_id $stage_id -class_id $class_id";
            if ($mode eq "goto_purged") {
                $command .= " -topurgedimfile";
            }
	    elsif ($mode eq "goto_cleaned") {
                $command .= " -tocleanedimfile";
            }
	    elsif ($mode eq "goto_scrubbed") {
		$command .= " -toscrubbedimfile";
	    }

            $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
            }
        } else {

	    # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
	    my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
            }
        }
    }
    exit 0;

} 
# Detrend stages
if ($stage eq "detrend.process.imfile") {

    die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id;
    ### select the imfiles for this entry

    # this stage uses 'dettool'
    my $dettool = can_run('dettool') or die "Can't find chiptool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $imfiles;                      # Array of component files
    my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
    }

    # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
    if (@$stdout_buf == 0)  {
	exit 0; # Silently exit if there's nothing to do.  I don't know how we'd ever get here, but let's be safe.
    }

    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    # extract the metadata for the files into a hash list
    $imfiles = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    # loop over all of the imfiles, determine the path_base and class_id for each
    foreach my $imfile (@$imfiles) {
	my $exp_id   = $imfile->{exp_id};
        my $class_id = $imfile->{class_id};
        my $path_base = $imfile->{path_base};
        my $status = 1;

        # don't clean up unless the data needed to update is available
        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
	# goto_scrubbed now requires the config file to not exist.
	
	# Possibly not the correct config file, but simtest doesn't leave any around to check.
        if ($mode eq "goto_cleaned") {
            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);

            if (!$config_file or ! -e $config_file) {
                print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id "
                    . " because config file is missing\n";
                $status = 0;
            }
        }
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);

	    if ($config_file and -e $config_file) {
		print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id "
		    . " because config file is present\n";
		$status = 0;
	    }
	}

        if ($status) {
            # array of actual filenames to delete
            my @files = ();

            # delete the temporary image datafiles
            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
            if ($mode eq "goto_purged") {
                # additional files to remove for 'purge' mode
		addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
		addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
            }

            # actual command to delete the files
            $status = &delete_files (\@files);
        }

        if ($status)  {
            my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile";
            if ($mode eq "goto_purged") {
                $command .= " -data_state purged";
            }
	    elsif ($mode eq "goto_cleaned") {
                $command .= " -data_state cleaned";
            }
	    elsif ($mode eq "goto_scrubbed") {
		$command .= " -data_state scrubbed";
	    }

            $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
            }

        } else {

	    # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
	    my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
            }
        }
    }

    # Check to see if we can mark the whole detRunSummary object as cleaned.

    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
    $command .= " -dbname $dbname" if defined $dbname;
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    }
    if (@$stdout_buf != 0) {
	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	my $exps = parse_md_list($metadata) or
	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	
	foreach my $exp (@$exps) {
	    my $iteration = $exp->{iteration};
	    my $command;
	    if ($mode eq "goto_cleaned") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
	    }
	}
    }
    
    exit 0;
}
if ($stage eq "detrend.process.exp") {
    die "--stage_id required for stage $stage\n" if !$stage_id;
    # this stage uses 'dettool'
    my $dettool = can_run('dettool') or die "Can't find dettool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $exps;                      # Array of component files
    my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    }
    
    if (@$stdout_buf == 0) {
	exit 0; #silently abort. I need to fix this for propers
    }

    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    $exps = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);


    foreach my $exp (@$exps) {
	my $path_base = $exp->{path_base};
	my $exp_id    = $exp->{exp_id};

	my $status = 1;
	# don't clean up unless the data needed to update is available
	# goto_scrubbed now requires the config file to not be present
	if ($mode eq "goto_cleaned") {
	    my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
	    
	    if (!$config_file or ! -e $config_file) {
		print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
		$status = 0;
	    }
	}
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
	    
	    if ($config_file and -e $config_file) {
		print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
		$status = 0;
	    }
	}
	if ($status) {
	    my @files = ();
	    # delete the temporary image datafiles
	    # I can't find anything to put here
	    if ($mode eq "goto_purged") {
		# additional files to remove for 'purge' mode
		addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
		addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
	    }
	    # actual command to delete the files
	    $status = &delete_files (\@files);
	}
	
	if ($status)  {
	    my $command;
	    if ($mode eq "goto_cleaned") {
		$command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	} else {
	    # since 'camera' has only a single imfile, we can just update the run
	    my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	    exit $PS_EXIT_UNKNOWN_ERROR;
	}
    }
    # Check to see if we can mark the whole detRunSummary object as cleaned.

    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
    $command .= " -dbname $dbname" if defined $dbname;
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    }
    if (@$stdout_buf != 0) {
	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	$exps = parse_md_list($metadata) or
	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	
	foreach my $exp (@$exps) {
	    my $iteration = $exp->{iteration};
	    my $command;
	    if ($mode eq "goto_cleaned") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
	    }
	}
    }

    exit 0;
}
if ($stage eq "detrend.stack.imfile") {
    
    die "--stage_id required for stage $stage\n" if !$stage_id;

    # this stage uses 'dettool'
    my $dettool = can_run('dettool') or die "Can't find dettool";

    # Get list of component imfiles
    my $stacks;                  # Array reference of component files
    my $command = "$dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    }
    my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 
	&my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    $stacks = parse_md_list($metadata) or 
	&my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    
    my @files = ();
    foreach my $stack (@{ $stacks }) {
	# detStackedImfile does not have a path_base column.  This is inconvenient, as it means we need to calculate it.
	my $path_base = $stack->{uri};
	my $iteration = $stack->{iteration};
	my $class_id  = $stack->{class_id};

	$path_base =~ s/\.fits$//; # That should do it?

	my $status = 1;

# 	if ($mode eq "goto_cleaned") {
# 	    my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);

# 	    $config_file =~ s%^file://%%;
# 	    if (!$config_file or ! -e $config_file) {
# 		print STDERR "skipping cleanup for $stage $stage_id $path_base" .
# 		    " because config file is missing\n";
# 		$status = 0;
# 	    }
# 	    $config_file = 'file://' . $config_file;
# 	}
# 	elsif ($mode eq "goto_scrubbed") {
# 	    my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
# 	    $config_file =~ s%^file://%%;
# 	    if ($config_file and -e $config_file) {
# 		print STDERR "skipping scrubbed for $stage $stage_id $path_base" .
# 		    " because config file is present\n";
# 		$status = 0;
# 	    }
# 	}

	if ($status) {
	    # delete the temporary image datafiles
	    # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion.
	    # I understand that they all point to the same filename right now, but that may not be true in
	    # the future.
	    addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id);
	    addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id);
	    addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id);
	    addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id);
	    addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id);
	    addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id);
	    

	    addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id);
	    addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id);

	    if ($mode eq "goto_purged") {
		# additional files to remove for 'purge' mode
#		addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id);
	    }

	    $status = &delete_files(\@files);
	}

	if ($status) {
	    my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id";
	    if ($mode eq "goto_purged") {
		$command .= " -updatestacked -data_state purged";
	    } 
	    elsif ($mode eq "goto_cleaned") {
		$command .= " -updatestacked -data_state cleaned";
	    }
	    elsif ($mode eq "goto_scrubbed") {
		$command .= " -updatestacked -data_state scrubbed";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	} else {
	    my $command = "$dettool -updatestacked  -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	    exit $PS_EXIT_UNKNOWN_ERROR;
	}
    }
    # Check to see if we can mark the whole detRunSummary object as cleaned.

    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
    $command .= " -dbname $dbname" if defined $dbname;
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    }
    if (@$stdout_buf != 0) {
	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	my $exps = parse_md_list($metadata) or
	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	
	foreach my $exp (@$exps) {
	    my $iteration = $exp->{iteration};
	    my $command;
	    if ($mode eq "goto_cleaned") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
	    }
	}
    }
    exit 0;
}
if ($stage eq "detrend.normstat.imfile") {
    print STDERR "I'm not convinced there's anything to clean up from stage $stage\n";
    die "--stage_id required for stage $stage\n" if !$stage_id;
    # this stage uses 'camtool'
    my $dettool = can_run('dettool') or die "Can't find dettool";

    my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    }
    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    my $exps = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    foreach my $exp (@$exps) {
#	my $path_base = $exp->{path_base};
	my $iteration = $exp->{iteration};
	my $class_id  = $exp->{class_id};

	my $status = 1;
	if ($status)  {
	    my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id";
	    if ($mode eq "goto_cleaned") {
		$command .= " -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command .= " -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command .= " -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		print STDERR "WTF: $success TTT $error_code QQQ\n";
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	} else {
	    my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	    exit $PS_EXIT_UNKNOWN_ERROR;
	}
    }
    # Check to see if we can mark the whole detRunSummary object as cleaned.

    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
    $command .= " -dbname $dbname" if defined $dbname;
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    }
    if (@$stdout_buf != 0) {
	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	$exps = parse_md_list($metadata) or
	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	
	foreach my $exp (@$exps) {
	    my $iteration = $exp->{iteration};
	    my $command;
	    if ($mode eq "goto_cleaned") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
	    }
	}
    }

    exit 0;

}
if ($stage eq "detrend.norm.imfile") {
    die "--stage_id required for stage $stage\n" if !$stage_id;
    # this stage uses 'dettool'
    my $dettool = can_run('dettool') or die "Can't find dettool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $exps;                      # Array of component files
    my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    }
    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    $exps = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    foreach my $exp (@$exps) {
	my $path_base = $exp->{path_base};
	my $iteration = $exp->{iteration};
	my $class_id  = $exp->{class_id};

	my $status = 1;
	# don't clean up unless the data needed to update is available
	# goto_scrubbed now requires the config file to not be present
	if ($mode eq "goto_cleaned") {
	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
	    
	    if (!$config_file or ! -e $config_file) {
		print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
		$status = 0;
	    }
	}
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
	    
	    if ($config_file and -e $config_file) {
		print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
		$status = 0;
	    }
	}
	if ($status) {
	    my @files = ();

	    if ($mode eq "goto_purged") {
		# additional files to remove for 'purge' mode
		addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base);
		addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base);

		addFilename (\@files, "PPIMAGE.OUTPUT", $path_base);
		addFilename (\@files, "PPIMAGE.STATS", $path_base);
	    }
	    # actual command to delete the files
	    $status = &delete_files (\@files);
	}
	
	if ($status)  {
	    my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id";
	    if ($mode eq "goto_cleaned") {
		$command .= " -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command .= " -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command .= " -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	} else {
	    my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	    exit $PS_EXIT_UNKNOWN_ERROR;
	}
    }
    # Check to see if we can mark the whole detRunSummary object as cleaned.

    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
    $command .= " -dbname $dbname" if defined $dbname;
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    }
    if (@$stdout_buf != 0) {
	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	$exps = parse_md_list($metadata) or
	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	
	foreach my $exp (@$exps) {
	    my $iteration = $exp->{iteration};
	    my $command;
	    if ($mode eq "goto_cleaned") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
	    }
	}
    }

    exit 0;
}
if ($stage eq "detrend.norm.exp") {
    die "--stage_id required for stage $stage\n" if !$stage_id;
    # this stage uses 'dettool'
    my $dettool = can_run('dettool') or die "Can't find dettool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $exps;                      # Array of component files
    my $command = "$dettool -pendingcleanup_normalizedexp -det_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    }

    if (@$stdout_buf == 0) {
	exit 0;
    }
    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    $exps = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    foreach my $exp (@$exps) {
	my $exp_id = $exp->{exp_id};
	my $iteration = $exp->{iteration};
	my $path_base = $exp->{path_base};

	my $status = 1;
	# don't clean up unless the data needed to update is available
	# goto_scrubbed now requires the config file to not be present
	if ($mode eq "goto_cleaned") {
	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
	    
	    if (!$config_file or ! -e $config_file) {
		print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
		$status = 0;
	    }
	}
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
	    
	    if ($config_file and -e $config_file) {
		print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
		$status = 0;
	    }
	}
	if ($status) {
	    my @files = ();
	    # delete the temporary image datafiles
	    if ($mode eq "goto_purged") {
		# additional files to remove for 'purge' mode
		addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
		addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
	    }
	    # actual command to delete the files
	    $status = &delete_files (\@files);
	}
	
	if ($status)  {
	    my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration";
	    if ($mode eq "goto_cleaned") {
		$command .= " -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command .= " -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command .= " -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		print STDERR " residexp had an issue setting the state:? $success $error_code\n";
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	} else {
	    my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	    exit $PS_EXIT_UNKNOWN_ERROR;
	}
    }
    # Check to see if we can mark the whole detRunSummary object as cleaned.

    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
    $command .= " -dbname $dbname" if defined $dbname;
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    }
    if (@$stdout_buf != 0) {
	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	$exps = parse_md_list($metadata) or
	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	
	foreach my $exp (@$exps) {
	    my $iteration = $exp->{iteration};
	    my $command;
	    if ($mode eq "goto_cleaned") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
	    }
	}
    }
    exit 0;
}
if ($stage eq "detrend.resid.imfile") {

    die "--stage_id required for stage $stage\n" if !$stage_id;
    ### select the imfiles for this entry

    # this stage uses 'dettool'
    my $dettool = can_run('dettool') or die "Can't find dettool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $imfiles;                      # Array of component files
    my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
    }

    if (@$stdout_buf == 0) {
	exit 0;
    }

    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    # extract the metadata for the files into a hash list
    $imfiles = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    # loop over all of the imfiles, determine the path_base and class_id for each
    foreach my $imfile (@$imfiles) {
        my $class_id = $imfile->{class_id};
	my $iteration = $imfile->{iteration};
	my $exp_id = $imfile->{exp_id};
        my $path_base = $imfile->{path_base};
        my $status = 1;

        # don't clean up unless the data needed to update is available
        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
	# goto_scrubbed now requires the config file to not exist.
	
	# Possibly not the correct config file, but simtest doesn't leave any around to check.
        if ($mode eq "goto_cleaned") {
            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);

            if (!$config_file or ! -e $config_file) {
                print STDERR "skipping cleanup for $stage $stage_id $class_id "
                    . " because config file is missing\n";
                $status = 0;
            }
        }
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);

	    if ($config_file and -e $config_file) {
		print STDERR "skipping scrubbed for $stage $stage_id $class_id "
		    . " because config file is present\n";
		$status = 0;
	    }
	}

        if ($status) {
            # array of actual filenames to delete
            my @files = ();

            # delete the temporary image datafiles
            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
            if ($mode eq "goto_purged") {
                # additional files to remove for 'purge' mode
		addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
		addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
            }

            # actual command to delete the files
            $status = &delete_files (\@files);
        }

        if ($status)  {
            my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id";
            if ($mode eq "goto_purged") {
                $command .= " -data_state purged";
            }
	    elsif ($mode eq "goto_cleaned") {
                $command .= " -data_state cleaned";
            }
	    elsif ($mode eq "goto_scrubbed") {
		$command .= " -data_state scrubbed";
	    }

            $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
            }
        } else {
	    my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;

            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
            }
        }
    }
    # Check to see if we can mark the whole detRunSummary object as cleaned.

    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
    $command .= " -dbname $dbname" if defined $dbname;
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    }
    if (@$stdout_buf != 0) {
	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	my $exps = parse_md_list($metadata) or
	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	
	foreach my $exp (@$exps) {
	    my $iteration = $exp->{iteration};
	    my $command;
	    if ($mode eq "goto_cleaned") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
	    }
	}
    }

    exit 0;
}
if ($stage eq "detrend.resid.exp") {
    die "--stage_id required for stage $stage\n" if !$stage_id;
    # this stage uses 'camtool'
    my $dettool = can_run('dettool') or die "Can't find dettool";

    # Get list of component imfiles
    # XXX may need a different my_die for each stage
    my $exps;                      # Array of component files
    my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run
    $command .= " -dbname $dbname" if defined $dbname;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    }
    # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this
    if (@$stdout_buf == 0) {
	exit 0;
    }
    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    $exps = parse_md_list($metadata) or
        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);

    foreach my $exp (@$exps) {
	my $exp_id = $exp->{exp_id};
	my $iteration = $exp->{iteration};
	my $path_base = $exp->{path_base};
#	my $class_id  = $exp->{class_id} 
	my $status = 1;
	# don't clean up unless the data needed to update is available
	# goto_scrubbed now requires the config file to not be present
	if ($mode eq "goto_cleaned") {
	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
	    
	    if (!$config_file or ! -e $config_file) {
		print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
		$status = 0;
	    }
	}
	elsif ($mode eq "goto_scrubbed") {
	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
	    
	    if ($config_file and -e $config_file) {
		print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
		$status = 0;
	    }
	}
	if ($status) {
	    my @files = ();
	    # delete the temporary image datafiles
	    if ($mode eq "goto_purged") {
		# additional files to remove for 'purge' mode
                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id);
                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id);
	    }
	    # actual command to delete the files
	    $status = &delete_files (\@files);
	}
	
	if ($status)  {
	    my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration";
	    if ($mode eq "goto_cleaned") {
		$command .= " -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command .= " -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command .= " -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		print STDERR " residexp had an issue setting the state:? $success $error_code\n";
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	} else {
	    my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
	    $command .= " -dbname $dbname" if defined $dbname;
	    
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
	    }
	    exit $PS_EXIT_UNKNOWN_ERROR;
	}
    }
    # Check to see if we can mark the whole detRunSummary object as cleaned.

    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
    $command .= " -dbname $dbname" if defined $dbname;
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    unless ($success) {
	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    }
    if (@$stdout_buf != 0) {
	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	$exps = parse_md_list($metadata) or
	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
	
	foreach my $exp (@$exps) {
	    my $iteration = $exp->{iteration};
	    my $command;
	    if ($mode eq "goto_cleaned") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
	    }
	    if ($mode eq "goto_scrubbed") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
	    }
	    if ($mode eq "goto_purged") {
		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
	    }
	    $command .= " -dbname $dbname" if defined $dbname;
	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
		run(command => $command, verbose => $verbose);
	    unless ($success) {
		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
	    }
	}
    }
    exit 0;
}


die "ipp_cleanup.pl -stage $stage not yet implemented\n";

sub delete_files
{
    my $files = shift; # reference to a list of files to unlink

#     open(TMPLOG,">>/tmp/czw.cleanup.log");
#     flock(TMPLOG,2);

    # this script is, of course, very dangerous.
    foreach my $file (@$files) {
        print STDERR "unlinking $file\n";
# 	print TMPLOG "$stage $stage_id $file";
# 	my $ff = $file;
# 	$ff =~ s%^file://%%;
# 	unless (-e $ff) {
# 	    print TMPLOG "\t File not found\n";
# 	}
# 	else {
# 	    print TMPLOG "\n";
# 	}

        $ipprc->file_delete($file);
    }

#     flock(TMPLOG,8);
#     close(TMPLOG);

    return 1;
}

sub addFilename
{
    my $files      = shift; # reference to a list of files to unlink
    my $filerule   = shift; # filerule to add
    my $path_base  = shift; # base filename
    my $class_id   = shift; # class_id, if needed

    my $file = $ipprc->filename($filerule, $path_base, $class_id);

    push @$files, $file;
    return 1;
}

# XXX we currently do not set the error state in the db on my_die
sub my_die
{
    my $msg = shift; # Warning message on die
    my $stage = shift; # stage name
    my $stage_id = shift; #  identifier
    my $exit_code = shift; # Exit code
    # outputImage and path_base are globals

    carp($msg);
    exit $exit_code;
}

__END__
