IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Sep 8, 2009, 5:32:55 PM (17 years ago)
Author:
watersc1
Message:

Merge my branch back into the trunk. This includes:

ipp_cleanup.pl : and associated edits to allow cleanup to work
burntool : updated to newest version, ipp_apply_burntool.pl modified to use persist=t options
ppImageBurntoolMask.c : masks region burntool identifies
psastro : allow selection of wcs header format
addtool : splits addstar step off from the camera stage

Location:
trunk
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk

  • trunk/ippScripts/scripts/ipp_cleanup.pl

    r24642 r25299  
    6060
    6161
    62 my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff  => 1,
    63                detrend.process.imfile => 1, detrend.process.exp => 1, detrend.stack.imfile => 1,
    64                detrend.normstat.imfile => 1, detrend.norm.imfile => 1, detrend.norm.exp => 1,
    65                detrend.resid.imfile => 1, detrend.resid.exp => 1 );
     62my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff"  => 1,
     63               "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1,
     64               "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1,
     65               "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 );
    6666unless ($stages{$stage}) {
    6767    die "unknown stage $stage for ipp_cleanup.pl\n";
    … …  
    119119        # don't clean up unless the data needed to update is available
    120120        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
     121        # goto_scrubbed now requires the config file to not exist.
    121122        if ($mode eq "goto_cleaned") {
    122123            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    123             print STDERR "CHIP: CONFIG_FILE : $config_file\n";
     124
    124125            if (!$config_file or ! -e $config_file) {
    125126                print STDERR "skipping cleanup for chipRun $stage_id $class_id "
    … …  
    128129            }
    129130        }
     131        elsif ($mode eq "goto_scrubbed") {
     132            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     133
     134            if ($config_file and -e $config_file) {
     135                print STDERR "skipping scrubbed for chipRun $stage_id $class_id "
     136                    . " because config file is present\n";
     137                $status = 0;
     138            }
     139        }
    130140
    131141        if ($status) {
    … …  
    160170            if ($mode eq "goto_purged") {
    161171                $command .= " -topurgedimfile";
    162             } else {
     172            }
     173            elsif ($mode eq "goto_cleaned") {
    163174                $command .= " -tocleanedimfile";
    164175            }
     176            elsif ($mode eq "goto_scrubbed") {
     177                $command .= " -toscrubbedimfile";
     178            }
     179
    165180            $command .= " -dbname $dbname" if defined $dbname;
    166181
    … …  
    218233    my $status = 1;
    219234    # don't clean up unless the data needed to update is available
     235    # goto_scrubbed now requires the config file to not be present
    220236    if ($mode eq "goto_cleaned") {
    221237        my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
    … …  
    225241            $status = 0;
    226242        }
     243    }
     244    elsif ($mode eq "goto_scrubbed") {
     245        my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
     246
     247        if ($config_file and -e $config_file) {
     248            print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n";
     249            $status = 0;
     250        }
    227251    }
    228252    if ($status) {
    … …  
    246270        }
    247271        if ($mode eq "goto_scrubbed") {
    248             $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";
     272            $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed";
    249273        }
    250274        if ($mode eq "goto_purged") {
    … …  
    311335            }
    312336        }
     337        elsif ($mode eq "goto_scrubbed") {
     338            my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);
     339
     340            if ($config_file and -e $config_file) {
     341                print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" .
     342                    " because config file is present\n";
     343                $status = 0;
     344            }
     345        }
    313346        if ($status) {
    314347            # delete the temporary image datafiles
    … …  
    337370            if ($mode eq "goto_purged") {
    338371                $command .= " -topurgedskyfile";
    339             } else {
     372            }
     373            elsif ($mode eq "goto_cleaned") {
    340374                $command .= " -tocleanedskyfile";
    341375            }
     376            elsif ($mode eq "goto_scrubbed") {
     377                $command .= " -toscrubbedskyfile";
     378            }
    342379            $command .= " -dbname $dbname" if defined $dbname;
    343380
    … …  
    395432        if ($mode eq "goto_cleaned") {
    396433            my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);
    397             print STDERR "MY CONFIG FILE = $config_file\n";
    398             printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file);
     434
    399435            $config_file =~ s%^file://%%;
    400436            if (!$config_file or ! -e $config_file) {
    … …  
    404440            }
    405441            $config_file = 'file://' . $config_file;
     442        }
     443        elsif ($mode eq "goto_scrubbed") {
     444            my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);
     445            $config_file =~ s%^file://%%;
     446            if ($config_file and -e $config_file) {
     447                print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" .
     448                    " because config file is present\n";
     449                $status = 0;
     450            }
    406451        }
    407452        if ($status) {
    … …  
    428473            if ($mode eq "goto_purged") {
    429474                $command .= " -updaterun -state purged";
    430             } else {
     475            }
     476            elsif ($mode eq "goto_cleaned") {
    431477                $command .= " -updaterun -state cleaned";
     478            }
     479            elsif ($mode eq "goto_scrubbed") {
     480                $command .= " -updaterun -state scrubbed";
    432481            }
    433482            $command .= " -dbname $dbname" if defined $dbname;
    … …  
    485534        if ($mode eq "goto_cleaned") {
    486535            my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
    487             print STDERR "MY CONFIG FILE = $config_file\n";
    488             printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file);
     536
    489537            $config_file =~ s%^file://%%;
    490538            if (!$config_file or ! -e $config_file) {
    … …  
    494542            }
    495543            $config_file = 'file://' . $config_file;
     544        }
     545        elsif ($mode eq "goto_scrubbed") {
     546            my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
     547            $config_file =~ s%^file://%%;
     548            if ($config_file and -e $config_file) {
     549                print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" .
     550                    " because config file ($config_file) is present\n";
     551                $status = 0;
     552            }
    496553        }
    497554        if ($status) {
    … …  
    529586        if ($status) {
    530587            my $command = "$difftool -diff_id $stage_id";
    531 #           my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id";
     588
    532589            if ($mode eq "goto_purged") {
    533590                $command .= " -updaterun -state purged";
    534             } else {
     591            }
     592            elsif ($mode eq "goto_cleaned") {
    535593                $command .= " -updaterun -state cleaned";
    536594            }
     595            elsif ($mode eq "goto_scrubbed") {
     596                $command .= " -updaterun -state scrubbed";
     597            }
     598
    537599            $command .= " -dbname $dbname" if defined $dbname;
    538600           
    … …  
    545607        } else {
    546608            my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state";
    547 #           my $command = "$difftool -updaterun -diff_id $stage_id -skycell_id $skycell_id -state $error_state";
     609
    548610            $command .= " -dbname $dbname" if defined $dbname;
    549611           
    … …  
    560622}
    561623if ($stage eq 'fake') {
    562     die "ipp_cleanup.pl -stage fake not yet implemented. Probably will just mark database cleaned.\n";
     624    print STDERR "This does not seem to work at present, as no files exist. Terminating quietly.\n";
     625    exit(0);
     626    die "--stage_id required for stage fake\n" if !$stage_id;
     627    ### select the imfiles for this entry
     628
     629    # this stage uses 'chiptool'
     630    my $faketool = can_run('faketool') or die "Can't find faketool";
     631
     632    # Get list of component imfiles
     633    # XXX may need a different my_die for each stage
     634    my $imfiles;                      # Array of component files
     635    my $command = "$faketool -pendingcleanupimfile -fake_id $stage_id"; # Command to run
     636    $command .= " -dbname $dbname" if defined $dbname;
     637    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     638    unless ($success) {
     639        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     640        &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
     641    }
     642
     643    # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
     644    if (@$stdout_buf == 0)  {
     645        my $command = "$faketool -fake_id $stage_id -updaterun -set_state new";
     646        $command .= " -dbname $dbname" if defined $dbname;
     647
     648        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     649            run(command => $command, verbose => $verbose);
     650        unless ($success) {
     651            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     652            &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
     653        }
     654        exit 0;
     655    }
     656
     657    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     658        &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR);
     659
     660    # extract the metadata for the files into a hash list
     661    $imfiles = parse_md_list($metadata) or
     662        &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR);
     663
     664    # loop over all of the imfiles, determine the path_base and class_id for each
     665    foreach my $imfile (@$imfiles) {
     666        my $class_id = $imfile->{class_id};
     667        my $path_base = $imfile->{path_base};
     668        my $status = 1;
     669
     670        # don't clean up unless the data needed to update is available
     671        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
     672        # goto_scrubbed now requires the config file to not exist.
     673        if ($mode eq "goto_cleaned") {
     674            my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id);
     675
     676            if (!$config_file or ! -e $config_file) {
     677                print STDERR "skipping cleanup for fakeRun $stage_id $class_id "
     678                    . " because config file is missing\n";
     679                $status = 0;
     680            }
     681        }
     682        elsif ($mode eq "goto_scrubbed") {
     683            my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id);
     684
     685            if ($config_file and -e $config_file) {
     686                print STDERR "skipping scrubbed for fakeRun $stage_id $class_id "
     687                    . " because config file is present\n";
     688                $status = 0;
     689            }
     690        }
     691
     692        if ($status) {
     693            # array of actual filenames to delete
     694            my @files = ();
     695
     696            # delete the temporary image datafiles
     697            addFilename (\@files, "PPSIM.OUTPUT.MEF", $path_base, $class_id);
     698            addFilename (\@files, "PPSIM.OUTPUT.SPL", $path_base, $class_id);
     699            addFilename (\@files, "PPSIM.FAKE.CHIP", $path_base, $class_id);
     700            addFilename (\@files, "PPSIM.FORCE.CHIP", $path_base, $class_id);
     701            if ($mode eq "goto_purged") {
     702                # additional files to remove for 'purge' mode
     703                addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id);
     704                addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id);
     705                addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id);
     706            }
     707
     708            # actual command to delete the files
     709            $status = &delete_files (\@files);
     710        }
     711
     712        if ($status)  {
     713            my $command = "$faketool -fake_id $stage_id -class_id $class_id";
     714            if ($mode eq "goto_purged") {
     715                $command .= " -topurgedimfile";
     716            }
     717            elsif ($mode eq "goto_cleaned") {
     718                $command .= " -tocleanedimfile";
     719            }
     720            elsif ($mode eq "goto_scrubbed") {
     721                $command .= " -toscrubbedimfile";
     722            }
     723
     724            $command .= " -dbname $dbname" if defined $dbname;
     725
     726            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     727                    run(command => $command, verbose => $verbose);
     728            unless ($success) {
     729                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     730                &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
     731            }
     732        } else {
     733
     734            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
     735            my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state";
     736            $command .= " -dbname $dbname" if defined $dbname;
     737
     738            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     739                    run(command => $command, verbose => $verbose);
     740            unless ($success) {
     741                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     742                &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
     743            }
     744        }
     745    }
     746    exit 0;
     747
    563748}
    564 # fake : faketool : -pendingcleanupimfile (loop over imfiles)
     749# Detrend stages
     750if ($stage eq "detrend.process.imfile") {
     751
     752    die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id;
     753    ### select the imfiles for this entry
     754
     755    # this stage uses 'dettool'
     756    my $dettool = can_run('dettool') or die "Can't find chiptool";
     757
     758    # Get list of component imfiles
     759    # XXX may need a different my_die for each stage
     760    my $imfiles;                      # Array of component files
     761    my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run
     762    $command .= " -dbname $dbname" if defined $dbname;
     763    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     764    unless ($success) {
     765        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     766        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
     767    }
     768
     769    # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
     770    if (@$stdout_buf == 0)  {
     771        exit 0; # Silently exit if there's nothing to do.  I don't know how we'd ever get here, but let's be safe.
     772    }
     773
     774    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     775        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     776
     777    # extract the metadata for the files into a hash list
     778    $imfiles = parse_md_list($metadata) or
     779        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     780
     781    # loop over all of the imfiles, determine the path_base and class_id for each
     782    foreach my $imfile (@$imfiles) {
     783        my $exp_id   = $imfile->{exp_id};
     784        my $class_id = $imfile->{class_id};
     785        my $path_base = $imfile->{path_base};
     786        my $status = 1;
     787
     788        # don't clean up unless the data needed to update is available
     789        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
     790        # goto_scrubbed now requires the config file to not exist.
     791       
     792        # Possibly not the correct config file, but simtest doesn't leave any around to check.
     793        if ($mode eq "goto_cleaned") {
     794            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     795
     796            if (!$config_file or ! -e $config_file) {
     797                print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id "
     798                    . " because config file is missing\n";
     799                $status = 0;
     800            }
     801        }
     802        elsif ($mode eq "goto_scrubbed") {
     803            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     804
     805            if ($config_file and -e $config_file) {
     806                print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id "
     807                    . " because config file is present\n";
     808                $status = 0;
     809            }
     810        }
     811
     812        if ($status) {
     813            # array of actual filenames to delete
     814            my @files = ();
     815
     816            # delete the temporary image datafiles
     817            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
     818            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
     819            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
     820            if ($mode eq "goto_purged") {
     821                # additional files to remove for 'purge' mode
     822                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     823                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
     824                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
     825            }
     826
     827            # actual command to delete the files
     828            $status = &delete_files (\@files);
     829        }
     830
     831        if ($status)  {
     832            my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile";
     833            if ($mode eq "goto_purged") {
     834                $command .= " -data_state purged";
     835            }
     836            elsif ($mode eq "goto_cleaned") {
     837                $command .= " -data_state cleaned";
     838            }
     839            elsif ($mode eq "goto_scrubbed") {
     840                $command .= " -data_state scrubbed";
     841            }
     842
     843            $command .= " -dbname $dbname" if defined $dbname;
     844
     845            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     846                    run(command => $command, verbose => $verbose);
     847            unless ($success) {
     848                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     849                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     850            }
     851
     852        } else {
     853
     854            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
     855            my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state";
     856            $command .= " -dbname $dbname" if defined $dbname;
     857
     858            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     859                    run(command => $command, verbose => $verbose);
     860            unless ($success) {
     861                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     862                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     863            }
     864        }
     865    }
     866
     867    # Check to see if we can mark the whole detRunSummary object as cleaned.
     868
     869    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     870    $command .= " -dbname $dbname" if defined $dbname;
     871    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     872    unless ($success) {
     873        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     874        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     875    }
     876    if (@$stdout_buf != 0) {
     877        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     878            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     879        my $exps = parse_md_list($metadata) or
     880            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     881       
     882        foreach my $exp (@$exps) {
     883            my $iteration = $exp->{iteration};
     884            my $command;
     885            if ($mode eq "goto_cleaned") {
     886                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     887            }
     888            if ($mode eq "goto_scrubbed") {
     889                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     890            }
     891            if ($mode eq "goto_purged") {
     892                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     893            }
     894            $command .= " -dbname $dbname" if defined $dbname;
     895            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     896                run(command => $command, verbose => $verbose);
     897            unless ($success) {
     898                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     899                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     900            }
     901        }
     902    }
     903   
     904    exit 0;
     905}
     906if ($stage eq "detrend.process.exp") {
     907    die "--stage_id required for stage $stage\n" if !$stage_id;
     908    # this stage uses 'dettool'
     909    my $dettool = can_run('dettool') or die "Can't find dettool";
     910
     911    # Get list of component imfiles
     912    # XXX may need a different my_die for each stage
     913    my $exps;                      # Array of component files
     914    my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run
     915    $command .= " -dbname $dbname" if defined $dbname;
     916    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     917    unless ($success) {
     918        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     919        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     920    }
     921   
     922    if (@$stdout_buf == 0) {
     923        exit 0; #silently abort. I need to fix this for propers
     924    }
     925
     926    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     927        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     928
     929    $exps = parse_md_list($metadata) or
     930        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     931
     932
     933    foreach my $exp (@$exps) {
     934        my $path_base = $exp->{path_base};
     935        my $exp_id    = $exp->{exp_id};
     936
     937        my $status = 1;
     938        # don't clean up unless the data needed to update is available
     939        # goto_scrubbed now requires the config file to not be present
     940        if ($mode eq "goto_cleaned") {
     941            my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
     942           
     943            if (!$config_file or ! -e $config_file) {
     944                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
     945                $status = 0;
     946            }
     947        }
     948        elsif ($mode eq "goto_scrubbed") {
     949            my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
     950           
     951            if ($config_file and -e $config_file) {
     952                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
     953                $status = 0;
     954            }
     955        }
     956        if ($status) {
     957            my @files = ();
     958            # delete the temporary image datafiles
     959            # I can't find anything to put here
     960            if ($mode eq "goto_purged") {
     961                # additional files to remove for 'purge' mode
     962                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
     963                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
     964            }
     965            # actual command to delete the files
     966            $status = &delete_files (\@files);
     967        }
     968       
     969        if ($status)  {
     970            my $command;
     971            if ($mode eq "goto_cleaned") {
     972                $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned";
     973            }
     974            if ($mode eq "goto_scrubbed") {
     975                $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed";
     976            }
     977            if ($mode eq "goto_purged") {
     978                $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged";
     979            }
     980            $command .= " -dbname $dbname" if defined $dbname;
     981            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     982                run(command => $command, verbose => $verbose);
     983            unless ($success) {
     984                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     985                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     986            }
     987        } else {
     988            # since 'camera' has only a single imfile, we can just update the run
     989            my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state";
     990            $command .= " -dbname $dbname" if defined $dbname;
     991           
     992            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     993                run(command => $command, verbose => $verbose);
     994            unless ($success) {
     995                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     996                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     997            }
     998            exit $PS_EXIT_UNKNOWN_ERROR;
     999        }
     1000    }
     1001    # Check to see if we can mark the whole detRunSummary object as cleaned.
     1002
     1003    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     1004    $command .= " -dbname $dbname" if defined $dbname;
     1005    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1006    unless ($success) {
     1007        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1008        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1009    }
     1010    if (@$stdout_buf != 0) {
     1011        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1012            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1013        $exps = parse_md_list($metadata) or
     1014            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1015       
     1016        foreach my $exp (@$exps) {
     1017            my $iteration = $exp->{iteration};
     1018            my $command;
     1019            if ($mode eq "goto_cleaned") {
     1020                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1021            }
     1022            if ($mode eq "goto_scrubbed") {
     1023                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1024            }
     1025            if ($mode eq "goto_purged") {
     1026                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1027            }
     1028            $command .= " -dbname $dbname" if defined $dbname;
     1029            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1030                run(command => $command, verbose => $verbose);
     1031            unless ($success) {
     1032                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1033                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1034            }
     1035        }
     1036    }
     1037
     1038    exit 0;
     1039}
     1040if ($stage eq "detrend.stack.imfile") {
     1041   
     1042    die "--stage_id required for stage $stage\n" if !$stage_id;
     1043
     1044    # this stage uses 'dettool'
     1045    my $dettool = can_run('dettool') or die "Can't find dettool";
     1046
     1047    # Get list of component imfiles
     1048    my $stacks;                  # Array reference of component files
     1049    my $command = "$dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run
     1050    $command .= " -dbname $dbname" if defined $dbname;
     1051    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1052    unless ($success) {
     1053        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1054        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1055    }
     1056    my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or
     1057        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1058
     1059    $stacks = parse_md_list($metadata) or
     1060        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1061   
     1062    my @files = ();
     1063    foreach my $stack (@{ $stacks }) {
     1064        # detStackedImfile does not have a path_base column.  This is inconvenient, as it means we need to calculate it.
     1065        my $path_base = $stack->{uri};
     1066        my $iteration = $stack->{iteration};
     1067        my $class_id  = $stack->{class_id};
     1068
     1069        $path_base =~ s/\.fits$//; # That should do it?
     1070
     1071        my $status = 1;
     1072
     1073#       if ($mode eq "goto_cleaned") {
     1074#           my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
     1075
     1076#           $config_file =~ s%^file://%%;
     1077#           if (!$config_file or ! -e $config_file) {
     1078#               print STDERR "skipping cleanup for $stage $stage_id $path_base" .
     1079#                   " because config file is missing\n";
     1080#               $status = 0;
     1081#           }
     1082#           $config_file = 'file://' . $config_file;
     1083#       }
     1084#       elsif ($mode eq "goto_scrubbed") {
     1085#           my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
     1086#           $config_file =~ s%^file://%%;
     1087#           if ($config_file and -e $config_file) {
     1088#               print STDERR "skipping scrubbed for $stage $stage_id $path_base" .
     1089#                   " because config file is present\n";
     1090#               $status = 0;
     1091#           }
     1092#       }
     1093
     1094        if ($status) {
     1095            # delete the temporary image datafiles
     1096            # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion.
     1097            # I understand that they all point to the same filename right now, but that may not be true in
     1098            # the future.
     1099            addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id);
     1100            addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id);
     1101            addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id);
     1102            addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id);
     1103            addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id);
     1104            addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id);
     1105           
     1106
     1107            addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id);
     1108            addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id);
     1109
     1110            if ($mode eq "goto_purged") {
     1111                # additional files to remove for 'purge' mode
     1112#               addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id);
     1113            }
     1114
     1115            $status = &delete_files(\@files);
     1116        }
     1117
     1118        if ($status) {
     1119            my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id";
     1120            if ($mode eq "goto_purged") {
     1121                $command .= " -updatestacked -data_state purged";
     1122            }
     1123            elsif ($mode eq "goto_cleaned") {
     1124                $command .= " -updatestacked -data_state cleaned";
     1125            }
     1126            elsif ($mode eq "goto_scrubbed") {
     1127                $command .= " -updatestacked -data_state scrubbed";
     1128            }
     1129            $command .= " -dbname $dbname" if defined $dbname;
     1130           
     1131            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1132                run(command => $command, verbose => $verbose);
     1133            unless ($success) {
     1134                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1135                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1136            }
     1137        } else {
     1138            my $command = "$dettool -updatestacked  -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1139            $command .= " -dbname $dbname" if defined $dbname;
     1140           
     1141            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1142                run(command => $command, verbose => $verbose);
     1143            unless ($success) {
     1144                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1145                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1146            }
     1147            exit $PS_EXIT_UNKNOWN_ERROR;
     1148        }
     1149    }
     1150    # Check to see if we can mark the whole detRunSummary object as cleaned.
     1151
     1152    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     1153    $command .= " -dbname $dbname" if defined $dbname;
     1154    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1155    unless ($success) {
     1156        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1157        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1158    }
     1159    if (@$stdout_buf != 0) {
     1160        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1161            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1162        my $exps = parse_md_list($metadata) or
     1163            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1164       
     1165        foreach my $exp (@$exps) {
     1166            my $iteration = $exp->{iteration};
     1167            my $command;
     1168            if ($mode eq "goto_cleaned") {
     1169                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1170            }
     1171            if ($mode eq "goto_scrubbed") {
     1172                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1173            }
     1174            if ($mode eq "goto_purged") {
     1175                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1176            }
     1177            $command .= " -dbname $dbname" if defined $dbname;
     1178            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1179                run(command => $command, verbose => $verbose);
     1180            unless ($success) {
     1181                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1182                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1183            }
     1184        }
     1185    }
     1186    exit 0;
     1187}
     1188if ($stage eq "detrend.normstat.imfile") {
     1189    print STDERR "I'm not convinced there's anything to clean up from stage $stage\n";
     1190    die "--stage_id required for stage $stage\n" if !$stage_id;
     1191    # this stage uses 'camtool'
     1192    my $dettool = can_run('dettool') or die "Can't find dettool";
     1193
     1194    my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run
     1195    $command .= " -dbname $dbname" if defined $dbname;
     1196    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1197    unless ($success) {
     1198        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1199        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1200    }
     1201    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1202        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1203
     1204    my $exps = parse_md_list($metadata) or
     1205        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1206
     1207    foreach my $exp (@$exps) {
     1208#       my $path_base = $exp->{path_base};
     1209        my $iteration = $exp->{iteration};
     1210        my $class_id  = $exp->{class_id};
     1211
     1212        my $status = 1;
     1213        if ($status)  {
     1214            my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id";
     1215            if ($mode eq "goto_cleaned") {
     1216                $command .= " -data_state cleaned";
     1217            }
     1218            if ($mode eq "goto_scrubbed") {
     1219                $command .= " -data_state scrubbed";
     1220            }
     1221            if ($mode eq "goto_purged") {
     1222                $command .= " -data_state purged";
     1223            }
     1224            $command .= " -dbname $dbname" if defined $dbname;
     1225            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1226                run(command => $command, verbose => $verbose);
     1227            unless ($success) {
     1228                print STDERR "WTF: $success TTT $error_code QQQ\n";
     1229                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1230                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1231            }
     1232        } else {
     1233            my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1234            $command .= " -dbname $dbname" if defined $dbname;
     1235           
     1236            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1237                run(command => $command, verbose => $verbose);
     1238            unless ($success) {
     1239                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1240                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1241            }
     1242            exit $PS_EXIT_UNKNOWN_ERROR;
     1243        }
     1244    }
     1245    # Check to see if we can mark the whole detRunSummary object as cleaned.
     1246
     1247    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     1248    $command .= " -dbname $dbname" if defined $dbname;
     1249    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1250    unless ($success) {
     1251        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1252        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1253    }
     1254    if (@$stdout_buf != 0) {
     1255        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1256            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1257        $exps = parse_md_list($metadata) or
     1258            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1259       
     1260        foreach my $exp (@$exps) {
     1261            my $iteration = $exp->{iteration};
     1262            my $command;
     1263            if ($mode eq "goto_cleaned") {
     1264                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1265            }
     1266            if ($mode eq "goto_scrubbed") {
     1267                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1268            }
     1269            if ($mode eq "goto_purged") {
     1270                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1271            }
     1272            $command .= " -dbname $dbname" if defined $dbname;
     1273            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1274                run(command => $command, verbose => $verbose);
     1275            unless ($success) {
     1276                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1277                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1278            }
     1279        }
     1280    }
     1281
     1282    exit 0;
     1283
     1284}
     1285if ($stage eq "detrend.norm.imfile") {
     1286    die "--stage_id required for stage $stage\n" if !$stage_id;
     1287    # this stage uses 'dettool'
     1288    my $dettool = can_run('dettool') or die "Can't find dettool";
     1289
     1290    # Get list of component imfiles
     1291    # XXX may need a different my_die for each stage
     1292    my $exps;                      # Array of component files
     1293    my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run
     1294    $command .= " -dbname $dbname" if defined $dbname;
     1295    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1296    unless ($success) {
     1297        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1298        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1299    }
     1300    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1301        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1302
     1303    $exps = parse_md_list($metadata) or
     1304        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1305
     1306    foreach my $exp (@$exps) {
     1307        my $path_base = $exp->{path_base};
     1308        my $iteration = $exp->{iteration};
     1309        my $class_id  = $exp->{class_id};
     1310
     1311        my $status = 1;
     1312        # don't clean up unless the data needed to update is available
     1313        # goto_scrubbed now requires the config file to not be present
     1314        if ($mode eq "goto_cleaned") {
     1315            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1316           
     1317            if (!$config_file or ! -e $config_file) {
     1318                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
     1319                $status = 0;
     1320            }
     1321        }
     1322        elsif ($mode eq "goto_scrubbed") {
     1323            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1324           
     1325            if ($config_file and -e $config_file) {
     1326                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
     1327                $status = 0;
     1328            }
     1329        }
     1330        if ($status) {
     1331            my @files = ();
     1332
     1333            if ($mode eq "goto_purged") {
     1334                # additional files to remove for 'purge' mode
     1335                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base);
     1336                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base);
     1337
     1338                addFilename (\@files, "PPIMAGE.OUTPUT", $path_base);
     1339                addFilename (\@files, "PPIMAGE.STATS", $path_base);
     1340            }
     1341            # actual command to delete the files
     1342            $status = &delete_files (\@files);
     1343        }
     1344       
     1345        if ($status)  {
     1346            my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id";
     1347            if ($mode eq "goto_cleaned") {
     1348                $command .= " -data_state cleaned";
     1349            }
     1350            if ($mode eq "goto_scrubbed") {
     1351                $command .= " -data_state scrubbed";
     1352            }
     1353            if ($mode eq "goto_purged") {
     1354                $command .= " -data_state purged";
     1355            }
     1356            $command .= " -dbname $dbname" if defined $dbname;
     1357            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1358                run(command => $command, verbose => $verbose);
     1359            unless ($success) {
     1360                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1361                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1362            }
     1363        } else {
     1364            my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1365            $command .= " -dbname $dbname" if defined $dbname;
     1366           
     1367            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1368                run(command => $command, verbose => $verbose);
     1369            unless ($success) {
     1370                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1371                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1372            }
     1373            exit $PS_EXIT_UNKNOWN_ERROR;
     1374        }
     1375    }
     1376    # Check to see if we can mark the whole detRunSummary object as cleaned.
     1377
     1378    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     1379    $command .= " -dbname $dbname" if defined $dbname;
     1380    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1381    unless ($success) {
     1382        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1383        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1384    }
     1385    if (@$stdout_buf != 0) {
     1386        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1387            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1388        $exps = parse_md_list($metadata) or
     1389            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1390       
     1391        foreach my $exp (@$exps) {
     1392            my $iteration = $exp->{iteration};
     1393            my $command;
     1394            if ($mode eq "goto_cleaned") {
     1395                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1396            }
     1397            if ($mode eq "goto_scrubbed") {
     1398                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1399            }
     1400            if ($mode eq "goto_purged") {
     1401                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1402            }
     1403            $command .= " -dbname $dbname" if defined $dbname;
     1404            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1405                run(command => $command, verbose => $verbose);
     1406            unless ($success) {
     1407                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1408                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1409            }
     1410        }
     1411    }
     1412
     1413    exit 0;
     1414}
     1415if ($stage eq "detrend.norm.exp") {
     1416    die "--stage_id required for stage $stage\n" if !$stage_id;
     1417    # this stage uses 'dettool'
     1418    my $dettool = can_run('dettool') or die "Can't find dettool";
     1419
     1420    # Get list of component imfiles
     1421    # XXX may need a different my_die for each stage
     1422    my $exps;                      # Array of component files
     1423    my $command = "$dettool -pendingcleanup_normalizedexp -det_id $stage_id"; # Command to run
     1424    $command .= " -dbname $dbname" if defined $dbname;
     1425    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1426    unless ($success) {
     1427        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1428        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1429    }
     1430
     1431    if (@$stdout_buf == 0) {
     1432        exit 0;
     1433    }
     1434    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1435        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1436
     1437    $exps = parse_md_list($metadata) or
     1438        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1439
     1440    foreach my $exp (@$exps) {
     1441        my $exp_id = $exp->{exp_id};
     1442        my $iteration = $exp->{iteration};
     1443        my $path_base = $exp->{path_base};
     1444
     1445        my $status = 1;
     1446        # don't clean up unless the data needed to update is available
     1447        # goto_scrubbed now requires the config file to not be present
     1448        if ($mode eq "goto_cleaned") {
     1449            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1450           
     1451            if (!$config_file or ! -e $config_file) {
     1452                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
     1453                $status = 0;
     1454            }
     1455        }
     1456        elsif ($mode eq "goto_scrubbed") {
     1457            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1458           
     1459            if ($config_file and -e $config_file) {
     1460                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
     1461                $status = 0;
     1462            }
     1463        }
     1464        if ($status) {
     1465            my @files = ();
     1466            # delete the temporary image datafiles
     1467            if ($mode eq "goto_purged") {
     1468                # additional files to remove for 'purge' mode
     1469                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
     1470                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
     1471            }
     1472            # actual command to delete the files
     1473            $status = &delete_files (\@files);
     1474        }
     1475       
     1476        if ($status)  {
     1477            my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration";
     1478            if ($mode eq "goto_cleaned") {
     1479                $command .= " -data_state cleaned";
     1480            }
     1481            if ($mode eq "goto_scrubbed") {
     1482                $command .= " -data_state scrubbed";
     1483            }
     1484            if ($mode eq "goto_purged") {
     1485                $command .= " -data_state purged";
     1486            }
     1487            $command .= " -dbname $dbname" if defined $dbname;
     1488            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1489                run(command => $command, verbose => $verbose);
     1490            unless ($success) {
     1491                print STDERR " residexp had an issue setting the state:? $success $error_code\n";
     1492                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1493                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1494            }
     1495        } else {
     1496            my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
     1497            $command .= " -dbname $dbname" if defined $dbname;
     1498           
     1499            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1500                run(command => $command, verbose => $verbose);
     1501            unless ($success) {
     1502                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1503                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1504            }
     1505            exit $PS_EXIT_UNKNOWN_ERROR;
     1506        }
     1507    }
     1508    # Check to see if we can mark the whole detRunSummary object as cleaned.
     1509
     1510    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     1511    $command .= " -dbname $dbname" if defined $dbname;
     1512    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1513    unless ($success) {
     1514        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1515        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1516    }
     1517    if (@$stdout_buf != 0) {
     1518        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1519            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1520        $exps = parse_md_list($metadata) or
     1521            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1522       
     1523        foreach my $exp (@$exps) {
     1524            my $iteration = $exp->{iteration};
     1525            my $command;
     1526            if ($mode eq "goto_cleaned") {
     1527                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1528            }
     1529            if ($mode eq "goto_scrubbed") {
     1530                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1531            }
     1532            if ($mode eq "goto_purged") {
     1533                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1534            }
     1535            $command .= " -dbname $dbname" if defined $dbname;
     1536            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1537                run(command => $command, verbose => $verbose);
     1538            unless ($success) {
     1539                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1540                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1541            }
     1542        }
     1543    }
     1544    exit 0;
     1545}
     1546if ($stage eq "detrend.resid.imfile") {
     1547
     1548    die "--stage_id required for stage $stage\n" if !$stage_id;
     1549    ### select the imfiles for this entry
     1550
     1551    # this stage uses 'dettool'
     1552    my $dettool = can_run('dettool') or die "Can't find dettool";
     1553
     1554    # Get list of component imfiles
     1555    # XXX may need a different my_die for each stage
     1556    my $imfiles;                      # Array of component files
     1557    my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run
     1558    $command .= " -dbname $dbname" if defined $dbname;
     1559    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1560    unless ($success) {
     1561        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1562        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
     1563    }
     1564
     1565    if (@$stdout_buf == 0) {
     1566        exit 0;
     1567    }
     1568
     1569    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1570        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1571
     1572    # extract the metadata for the files into a hash list
     1573    $imfiles = parse_md_list($metadata) or
     1574        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1575
     1576    # loop over all of the imfiles, determine the path_base and class_id for each
     1577    foreach my $imfile (@$imfiles) {
     1578        my $class_id = $imfile->{class_id};
     1579        my $iteration = $imfile->{iteration};
     1580        my $exp_id = $imfile->{exp_id};
     1581        my $path_base = $imfile->{path_base};
     1582        my $status = 1;
     1583
     1584        # don't clean up unless the data needed to update is available
     1585        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
     1586        # goto_scrubbed now requires the config file to not exist.
     1587       
     1588        # Possibly not the correct config file, but simtest doesn't leave any around to check.
     1589        if ($mode eq "goto_cleaned") {
     1590            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     1591
     1592            if (!$config_file or ! -e $config_file) {
     1593                print STDERR "skipping cleanup for $stage $stage_id $class_id "
     1594                    . " because config file is missing\n";
     1595                $status = 0;
     1596            }
     1597        }
     1598        elsif ($mode eq "goto_scrubbed") {
     1599            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     1600
     1601            if ($config_file and -e $config_file) {
     1602                print STDERR "skipping scrubbed for $stage $stage_id $class_id "
     1603                    . " because config file is present\n";
     1604                $status = 0;
     1605            }
     1606        }
     1607
     1608        if ($status) {
     1609            # array of actual filenames to delete
     1610            my @files = ();
     1611
     1612            # delete the temporary image datafiles
     1613            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
     1614            if ($mode eq "goto_purged") {
     1615                # additional files to remove for 'purge' mode
     1616                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     1617                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
     1618                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
     1619            }
     1620
     1621            # actual command to delete the files
     1622            $status = &delete_files (\@files);
     1623        }
     1624
     1625        if ($status)  {
     1626            my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id";
     1627            if ($mode eq "goto_purged") {
     1628                $command .= " -data_state purged";
     1629            }
     1630            elsif ($mode eq "goto_cleaned") {
     1631                $command .= " -data_state cleaned";
     1632            }
     1633            elsif ($mode eq "goto_scrubbed") {
     1634                $command .= " -data_state scrubbed";
     1635            }
     1636
     1637            $command .= " -dbname $dbname" if defined $dbname;
     1638
     1639            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1640                    run(command => $command, verbose => $verbose);
     1641            unless ($success) {
     1642                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1643                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1644            }
     1645        } else {
     1646            my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1647            $command .= " -dbname $dbname" if defined $dbname;
     1648
     1649            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1650                    run(command => $command, verbose => $verbose);
     1651            unless ($success) {
     1652                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1653                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1654            }
     1655        }
     1656    }
     1657    # Check to see if we can mark the whole detRunSummary object as cleaned.
     1658
     1659    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     1660    $command .= " -dbname $dbname" if defined $dbname;
     1661    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1662    unless ($success) {
     1663        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1664        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1665    }
     1666    if (@$stdout_buf != 0) {
     1667        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1668            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1669        my $exps = parse_md_list($metadata) or
     1670            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1671       
     1672        foreach my $exp (@$exps) {
     1673            my $iteration = $exp->{iteration};
     1674            my $command;
     1675            if ($mode eq "goto_cleaned") {
     1676                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1677            }
     1678            if ($mode eq "goto_scrubbed") {
     1679                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1680            }
     1681            if ($mode eq "goto_purged") {
     1682                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1683            }
     1684            $command .= " -dbname $dbname" if defined $dbname;
     1685            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1686                run(command => $command, verbose => $verbose);
     1687            unless ($success) {
     1688                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1689                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1690            }
     1691        }
     1692    }
     1693
     1694    exit 0;
     1695}
     1696if ($stage eq "detrend.resid.exp") {
     1697    die "--stage_id required for stage $stage\n" if !$stage_id;
     1698    # this stage uses 'camtool'
     1699    my $dettool = can_run('dettool') or die "Can't find dettool";
     1700
     1701    # Get list of component imfiles
     1702    # XXX may need a different my_die for each stage
     1703    my $exps;                      # Array of component files
     1704    my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run
     1705    $command .= " -dbname $dbname" if defined $dbname;
     1706    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1707    unless ($success) {
     1708        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1709        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1710    }
     1711    # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this
     1712    if (@$stdout_buf == 0) {
     1713        exit 0;
     1714    }
     1715    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1716        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1717
     1718    $exps = parse_md_list($metadata) or
     1719        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1720
     1721    foreach my $exp (@$exps) {
     1722        my $exp_id = $exp->{exp_id};
     1723        my $iteration = $exp->{iteration};
     1724        my $path_base = $exp->{path_base};
     1725#       my $class_id  = $exp->{class_id}
     1726        my $status = 1;
     1727        # don't clean up unless the data needed to update is available
     1728        # goto_scrubbed now requires the config file to not be present
     1729        if ($mode eq "goto_cleaned") {
     1730            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1731           
     1732            if (!$config_file or ! -e $config_file) {
     1733                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
     1734                $status = 0;
     1735            }
     1736        }
     1737        elsif ($mode eq "goto_scrubbed") {
     1738            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1739           
     1740            if ($config_file and -e $config_file) {
     1741                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
     1742                $status = 0;
     1743            }
     1744        }
     1745        if ($status) {
     1746            my @files = ();
     1747            # delete the temporary image datafiles
     1748            if ($mode eq "goto_purged") {
     1749                # additional files to remove for 'purge' mode
     1750                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id);
     1751                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id);
     1752            }
     1753            # actual command to delete the files
     1754            $status = &delete_files (\@files);
     1755        }
     1756       
     1757        if ($status)  {
     1758            my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration";
     1759            if ($mode eq "goto_cleaned") {
     1760                $command .= " -data_state cleaned";
     1761            }
     1762            if ($mode eq "goto_scrubbed") {
     1763                $command .= " -data_state scrubbed";
     1764            }
     1765            if ($mode eq "goto_purged") {
     1766                $command .= " -data_state purged";
     1767            }
     1768            $command .= " -dbname $dbname" if defined $dbname;
     1769            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1770                run(command => $command, verbose => $verbose);
     1771            unless ($success) {
     1772                print STDERR " residexp had an issue setting the state:? $success $error_code\n";
     1773                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1774                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1775            }
     1776        } else {
     1777            my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
     1778            $command .= " -dbname $dbname" if defined $dbname;
     1779           
     1780            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1781                run(command => $command, verbose => $verbose);
     1782            unless ($success) {
     1783                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1784                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1785            }
     1786            exit $PS_EXIT_UNKNOWN_ERROR;
     1787        }
     1788    }
     1789    # Check to see if we can mark the whole detRunSummary object as cleaned.
     1790
     1791    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     1792    $command .= " -dbname $dbname" if defined $dbname;
     1793    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1794    unless ($success) {
     1795        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1796        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1797    }
     1798    if (@$stdout_buf != 0) {
     1799        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1800            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1801        $exps = parse_md_list($metadata) or
     1802            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1803       
     1804        foreach my $exp (@$exps) {
     1805            my $iteration = $exp->{iteration};
     1806            my $command;
     1807            if ($mode eq "goto_cleaned") {
     1808                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1809            }
     1810            if ($mode eq "goto_scrubbed") {
     1811                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1812            }
     1813            if ($mode eq "goto_purged") {
     1814                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1815            }
     1816            $command .= " -dbname $dbname" if defined $dbname;
     1817            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1818                run(command => $command, verbose => $verbose);
     1819            unless ($success) {
     1820                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1821                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1822            }
     1823        }
     1824    }
     1825    exit 0;
     1826}
     1827
    5651828
    5661829die "ipp_cleanup.pl -stage $stage not yet implemented\n";
    … …  
    5701833    my $files = shift; # reference to a list of files to unlink
    5711834
     1835#     open(TMPLOG,">>/tmp/czw.cleanup.log");
     1836#     flock(TMPLOG,2);
     1837
    5721838    # this script is, of course, very dangerous.
    5731839    foreach my $file (@$files) {
    5741840        print STDERR "unlinking $file\n";
     1841#       print TMPLOG "$stage $stage_id $file";
     1842#       my $ff = $file;
     1843#       $ff =~ s%^file://%%;
     1844#       unless (-e $ff) {
     1845#           print TMPLOG "\t File not found\n";
     1846#       }
     1847#       else {
     1848#           print TMPLOG "\n";
     1849#       }
     1850
    5751851        $ipprc->file_delete($file);
    5761852    }
     1853
     1854#     flock(TMPLOG,8);
     1855#     close(TMPLOG);
     1856
    5771857    return 1;
    5781858}
Note: See TracChangeset for help on using the changeset viewer.