IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 26939


Ignore:
Timestamp:
Feb 12, 2010, 5:00:35 PM (16 years ago)
Author:
watersc1
Message:

Oops, that's almost certainly my typo.

Location:
trunk/ippScripts/scripts
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/diff_skycell.pl

    r26929 r26939  
    1 >>#!/usr/bin/env perl
     1#!/usr/bin/env perl
    22
    33use warnings;

  • trunk/ippScripts/scripts/ipp_cleanup.pl

    r26549 r26939  
    6161
    6262my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff"  => 1,
    63                "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1,
    64                "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1,
    65                "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 );
    66 unless ($stages{$stage}) {
     63               "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0,
     64               "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 );
     65unless (exists($stages{$stage})) {
    6766    die "unknown stage $stage for ipp_cleanup.pl\n";
    6867}
     68unless (($stages{$stage})) {
     69    die "unimplemented stage $stage for ipp_cleanup.pl\n";
     70}
     71
    6972
    7073my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    … …  
    888891}
    889892# Detrend stages
    890 if ($stage eq "detrend.process.imfile") {
     893if ($stage eq "detrend.processed") {
    891894
    892895    die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id;
    893896    ### select the imfiles for this entry
     897
     898    # Neither det_id nor exp_id uniquely determine a det exposure, so we pack them.
     899    my ($det_id,$exp_id) = split /\./, $stage_id;       #/ trailing slash for emacs;
    894900
    895901    # this stage uses 'dettool'
    … …  
    899905    # XXX may need a different my_die for each stage
    900906    my $imfiles;                      # Array of component files
    901     my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run
     907    my $metadata;
     908    my $command = "$dettool -pendingcleanup_processedimfile -det_id $det_id -exp_id $exp_id"; # Command to run
    902909    $command .= " -dbname $dbname" if defined $dbname;
    903910    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    904911    unless ($success) {
    905912        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    906         &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
     913        &my_die("Unable to perform dettool: $error_code", "detrend.processed", $stage_id, $error_code);
    907914    }
    908915
    909916    # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
    910     if (@$stdout_buf == 0)  {
    911         exit 0; # Silently exit if there's nothing to do.  I don't know how we'd ever get here, but let's be safe.
    912     }
    913 
    914     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     917    if (@$stdout_buf != 0)  {
     918#       exit 0; # Silently exit if there's nothing to do.  I don't know how we'd ever get here, but let's be safe.
     919
     920
     921        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    915922        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    916 
    917     # extract the metadata for the files into a hash list
    918     $imfiles = parse_md_list($metadata) or
    919         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    920 
     923       
     924        # extract the metadata for the files into a hash list
     925        $imfiles = parse_md_list($metadata) or
     926            &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     927    }
    921928    # loop over all of the imfiles, determine the path_base and class_id for each
    922929    foreach my $imfile (@$imfiles) {
    923         my $exp_id   = $imfile->{exp_id};
     930        my $iexp_id   = $imfile->{exp_id};
    924931        my $class_id = $imfile->{class_id};
    925932        my $path_base = $imfile->{path_base};
    926933        my $status = 1;
    927934
    928         # don't clean up unless the data needed to update is available
    929         # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
    930         # goto_scrubbed now requires the config file to not exist.
    931        
    932         # Possibly not the correct config file, but simtest doesn't leave any around to check.
    933         if ($mode eq "goto_cleaned") {
    934             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    935 
    936             unless ($ipprc->file_exists($config_file)) {
    937                 print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id "
    938                     . " because config file is missing\n";
    939                 $status = 0;
    940             }
    941         }
    942         elsif ($mode eq "goto_scrubbed") {
    943             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    944 
    945             if ($ipprc->file_exists($config_file)) {
    946                 print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id "
    947                     . " because config file is present\n";
    948                 $status = 0;
    949             }
    950         }
    951 
     935        unless (defined($path_base)) {
     936            print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n";
     937            $status = 0;
     938        }
     939        unless (defined($class_id)) {
     940            print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n";
     941            $status = 0;
     942        }
     943        # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent,
     944        # and so there is no check for config files.
    952945        if ($status) {
    953946            # array of actual filenames to delete
    954947            my @files = ();
    955 
    956948            # delete the temporary image datafiles
    957949            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
    … …  
    964956                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
    965957            }
    966 
    967958            # actual command to delete the files
    968959            $status = &delete_files (\@files);
    … …  
    970961
    971962        if ($status)  {
    972             my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile";
     963            my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile";
    973964            if ($mode eq "goto_purged") {
    974965                $command .= " -data_state purged";
    … …  
    980971                $command .= " -data_state scrubbed";
    981972            }
    982 
    983973            $command .= " -dbname $dbname" if defined $dbname;
    984974
    … …  
    991981
    992982        } else {
    993 
    994             # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
    995             my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state";
     983            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_*
     984            my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile ";
     985            $command .= " -data_state $error_state";
    996986            $command .= " -dbname $dbname" if defined $dbname;
    997987
    … …  
    1002992                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1003993            }
    1004         }
    1005     }
    1006 
    1007     # Check to see if we can mark the whole detRunSummary object as cleaned.
    1008 
    1009     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     994        }
     995    }
     996
     997    # Flag the detProcessedExp as clean now as well, if it is marked to be cleaned (this is clunky, but works for now).
     998
     999    $command = "$dettool -pendingcleanup_processedexp -det_id $det_id -exp_id $exp_id";
    10101000    $command .= " -dbname $dbname" if defined $dbname;
    10111001    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    10121002    unless ($success) {
    10131003        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1014         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1004        &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code);
    10151005    }
    10161006    if (@$stdout_buf != 0) {
    10171007        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1018             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1008            &my_die("Unable to parse metadata config doc", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR);
    10191009        my $exps = parse_md_list($metadata) or
    1020             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1010            &my_die("Unable to parse metadata list", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR);
    10211011       
    10221012        foreach my $exp (@$exps) {
    1023             my $iteration = $exp->{iteration};
    1024             my $command;
     1013            my $exp_id = $exp->{exp_id};
     1014            my $command = "$dettool -updateprocessedexp -det_id $det_id -exp_id $exp_id ";
    10251015            if ($mode eq "goto_cleaned") {
    1026                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1016                $command .= " -data_state cleaned ";
    10271017            }
    10281018            if ($mode eq "goto_scrubbed") {
    1029                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1019                $command .= " -data_state scrubbed ";
    10301020            }
    10311021            if ($mode eq "goto_purged") {
    1032                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1033             }
    1034             $command .= " -dbname $dbname" if defined $dbname;
     1022                $command .= " -data_state purged ";
     1023            }
     1024            $command .= " -dbname $dbname" if defined $dbname;
     1025#           print "$command\n";
    10351026            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    10361027                run(command => $command, verbose => $verbose);
    10371028            unless ($success) {
    10381029                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1039                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1040             }
    1041         }
    1042     }
    1043    
     1030                &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code);
     1031            }
     1032        }
     1033    }
     1034     
    10441035    exit 0;
    10451036}
    1046 if ($stage eq "detrend.process.exp") {
     1037if ($stage eq "detrend.resid") {
     1038
    10471039    die "--stage_id required for stage $stage\n" if !$stage_id;
     1040    ### select the imfiles for this entry
     1041
     1042    # Neither det_id nor exp_id uniquely determine the det exposure, so we pack them
     1043    my ($det_id,$exp_id) = split /\./, $stage_id;          #/ trailing slash for emacs;
     1044
    10481045    # this stage uses 'dettool'
    10491046    my $dettool = can_run('dettool') or die "Can't find dettool";
    … …  
    10511048    # Get list of component imfiles
    10521049    # XXX may need a different my_die for each stage
    1053     my $exps;                      # Array of component files
    1054     my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run
     1050    my $imfiles;                      # Array of component files
     1051    my $metadata;
     1052    my $command = "$dettool -pendingcleanup_residimfile -det_id $det_id -exp_id $exp_id"; # Command to run
    10551053    $command .= " -dbname $dbname" if defined $dbname;
    10561054    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    10571055    unless ($success) {
    10581056        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1059         &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1057        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
    10601058    }
    10611059   
    1062     if (@$stdout_buf == 0) {
    1063         exit 0; #silently abort. I need to fix this for propers
    1064     }
    1065 
    1066     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1067         &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1068 
    1069     $exps = parse_md_list($metadata) or
    1070         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1071 
    1072 
    1073     foreach my $exp (@$exps) {
    1074         my $path_base = $exp->{path_base};
    1075         my $exp_id    = $exp->{exp_id};
    1076 
    1077         my $status = 1;
    1078         # don't clean up unless the data needed to update is available
    1079         # goto_scrubbed now requires the config file to not be present
    1080         if ($mode eq "goto_cleaned") {
    1081             my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
    1082            
    1083             unless ($ipprc->file_exists($config_file)) {
    1084                 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
    1085                 $status = 0;
    1086             }
    1087         }
    1088         elsif ($mode eq "goto_scrubbed") {
    1089             my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
    1090            
    1091             if ($ipprc->file_exists($config_file)) {
    1092                 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
    1093                 $status = 0;
    1094             }
    1095         }
    1096         if ($status) {
    1097             my @files = ();
    1098             # delete the temporary image datafiles
    1099             # I can't find anything to put here
    1100             if ($mode eq "goto_purged") {
    1101                 # additional files to remove for 'purge' mode
    1102                 addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
    1103                 addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
    1104             }
    1105             # actual command to delete the files
    1106             $status = &delete_files (\@files);
    1107         }
     1060    # if there are no detResidImfiles (@$stdout_buf == 0), then silently exit.
     1061    if (@$stdout_buf != 0) {
     1062        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1063            &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    11081064       
    1109         if ($status)  {
    1110             my $command;
    1111             if ($mode eq "goto_cleaned") {
    1112                 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned";
    1113             }
    1114             if ($mode eq "goto_scrubbed") {
    1115                 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed";
    1116             }
    1117             if ($mode eq "goto_purged") {
    1118                 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged";
    1119             }
    1120             $command .= " -dbname $dbname" if defined $dbname;
    1121             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1122                 run(command => $command, verbose => $verbose);
    1123             unless ($success) {
    1124                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1125                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1126             }
    1127         } else {
    1128             # since 'camera' has only a single imfile, we can just update the run
    1129             my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state";
    1130             $command .= " -dbname $dbname" if defined $dbname;
    1131            
    1132             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1133                 run(command => $command, verbose => $verbose);
    1134             unless ($success) {
    1135                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1136                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1137             }
    1138             exit $PS_EXIT_UNKNOWN_ERROR;
    1139         }
    1140     }
    1141     # Check to see if we can mark the whole detRunSummary object as cleaned.
    1142 
    1143     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     1065        # extract the metadata for the files into a hash list
     1066        $imfiles = parse_md_list($metadata) or
     1067            &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1068    }
     1069    # loop over all of the imfiles, determine the path_base and class_id for each
     1070    foreach my $imfile (@$imfiles) {
     1071        my $iexp_id = $imfile->{exp_id};
     1072        my $class_id = $imfile->{class_id};
     1073        my $path_base = $imfile->{path_base};
     1074        my $iteration = $imfile->{iteration};
     1075
     1076        my $status = 1;
     1077
     1078        # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent,
     1079        # and so there is no check for config files.
     1080        unless (defined($path_base)) {
     1081            print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n";
     1082            $status = 0;
     1083        }
     1084        unless (defined($class_id)) {
     1085            print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n";
     1086            $status = 0;
     1087        }
     1088        if ($status) {
     1089            # array of actual filenames to delete
     1090            my @files = ();
     1091
     1092            # delete the temporary image datafiles
     1093            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
     1094            if ($mode eq "goto_purged") {
     1095                # additional files to remove for 'purge' mode
     1096                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     1097                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
     1098                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
     1099            }
     1100#           foreach my $f (@files) {
     1101#               print "RESID: $f\n";
     1102#           }
     1103            # actual command to delete the files
     1104            $status = &delete_files (\@files);
     1105        }
     1106
     1107        if ($status)  {
     1108            my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile ";
     1109            if ($mode eq "goto_purged") {
     1110                $command .= " -data_state purged";
     1111            }
     1112            elsif ($mode eq "goto_cleaned") {
     1113                $command .= " -data_state cleaned";
     1114            }
     1115            elsif ($mode eq "goto_scrubbed") {
     1116                $command .= " -data_state scrubbed";
     1117            }
     1118
     1119            $command .= " -dbname $dbname" if defined $dbname;
     1120
     1121            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1122                    run(command => $command, verbose => $verbose);
     1123            unless ($success) {
     1124                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1125                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1126            }
     1127        } else {
     1128            my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile ";
     1129            $command .= " -data_state $error_state ";
     1130            $command .= " -dbname $dbname" if defined $dbname;
     1131
     1132            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1133                    run(command => $command, verbose => $verbose);
     1134            unless ($success) {
     1135                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1136                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1137            }
     1138        }
     1139    }
     1140    # Flag the detResidExp as clean now as well, if it is marked tobe cleaned (this is still clunky).
     1141
     1142    $command = "$dettool -pendingcleanup_residexp -det_id $det_id -exp_id $exp_id";
    11441143    $command .= " -dbname $dbname" if defined $dbname;
    11451144    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    11461145    unless ($success) {
    11471146        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1148         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1147        &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code);
    11491148    }
    11501149    if (@$stdout_buf != 0) {
    11511150        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1152             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1153         $exps = parse_md_list($metadata) or
    1154             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1151            &my_die("Unable to parse metadata config doc", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR);
     1152        my $exps = parse_md_list($metadata) or
     1153            &my_die("Unable to parse metadata list", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR);
    11551154       
    11561155        foreach my $exp (@$exps) {
    11571156            my $iteration = $exp->{iteration};
    1158             my $command;
     1157            my $command = "$dettool -updateresidexp -det_id $det_id -exp_id $exp_id ";
    11591158            if ($mode eq "goto_cleaned") {
    1160                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1159                $command .= " -data_state cleaned";
    11611160            }
    11621161            if ($mode eq "goto_scrubbed") {
    1163                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1162                $command .= " -data_state scrubbed";
    11641163            }
    11651164            if ($mode eq "goto_purged") {
    1166                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1165                $command .= " -data_state purged";
    11671166            }
    11681167            $command .= " -dbname $dbname" if defined $dbname;
    … …  
    11711170            unless ($success) {
    11721171                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1173                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1172                &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code);
    11741173            }
    11751174        }
    … …  
    11781177    exit 0;
    11791178}
     1179
    11801180if ($stage eq "detrend.stack.imfile") {
    11811181   
    … …  
    12101210
    12111211        my $status = 1;
    1212 
    1213 #       if ($mode eq "goto_cleaned") {
    1214 #           my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
    1215 
    1216 #           $config_file =~ s%^file://%%;
    1217 #           if (!$config_file or ! -e $config_file) {
    1218 #               print STDERR "skipping cleanup for $stage $stage_id $path_base" .
    1219 #                   " because config file is missing\n";
    1220 #               $status = 0;
    1221 #           }
    1222 #           $config_file = 'file://' . $config_file;
    1223 #       }
    1224 #       elsif ($mode eq "goto_scrubbed") {
    1225 #           my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
    1226 #           $config_file =~ s%^file://%%;
    1227 #           if ($config_file and -e $config_file) {
    1228 #               print STDERR "skipping scrubbed for $stage $stage_id $path_base" .
    1229 #                   " because config file is present\n";
    1230 #               $status = 0;
    1231 #           }
    1232 #       }
    12331212
    12341213        if ($status) {
    … …  
    16841663    exit 0;
    16851664}
    1686 if ($stage eq "detrend.resid.imfile") {
    1687 
    1688     die "--stage_id required for stage $stage\n" if !$stage_id;
    1689     ### select the imfiles for this entry
    1690 
    1691     # this stage uses 'dettool'
    1692     my $dettool = can_run('dettool') or die "Can't find dettool";
    1693 
    1694     # Get list of component imfiles
    1695     # XXX may need a different my_die for each stage
    1696     my $imfiles;                      # Array of component files
    1697     my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run
    1698     $command .= " -dbname $dbname" if defined $dbname;
    1699     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    1700     unless ($success) {
    1701         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1702         &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
    1703     }
    1704 
    1705     if (@$stdout_buf == 0) {
    1706         exit 0;
    1707     }
    1708 
    1709     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1710         &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1711 
    1712     # extract the metadata for the files into a hash list
    1713     $imfiles = parse_md_list($metadata) or
    1714         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1715 
    1716     # loop over all of the imfiles, determine the path_base and class_id for each
    1717     foreach my $imfile (@$imfiles) {
    1718         my $class_id = $imfile->{class_id};
    1719         my $iteration = $imfile->{iteration};
    1720         my $exp_id = $imfile->{exp_id};
    1721         my $path_base = $imfile->{path_base};
    1722         my $status = 1;
    1723 
    1724         # don't clean up unless the data needed to update is available
    1725         # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
    1726         # goto_scrubbed now requires the config file to not exist.
    1727        
    1728         # Possibly not the correct config file, but simtest doesn't leave any around to check.
    1729         if ($mode eq "goto_cleaned") {
    1730             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    1731 
    1732             unless ($ipprc->file_exists($config_file)) {
    1733                 print STDERR "skipping cleanup for $stage $stage_id $class_id "
    1734                     . " because config file is missing\n";
    1735                 $status = 0;
    1736             }
    1737         }
    1738         elsif ($mode eq "goto_scrubbed") {
    1739             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    1740 
    1741             if ($ipprc->file_exists($config_file)) {
    1742                 print STDERR "skipping scrubbed for $stage $stage_id $class_id "
    1743                     . " because config file is present\n";
    1744                 $status = 0;
    1745             }
    1746         }
    1747 
    1748         if ($status) {
    1749             # array of actual filenames to delete
    1750             my @files = ();
    1751 
    1752             # delete the temporary image datafiles
    1753             addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
    1754             if ($mode eq "goto_purged") {
    1755                 # additional files to remove for 'purge' mode
    1756                 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
    1757                 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
    1758                 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
    1759             }
    1760 
    1761             # actual command to delete the files
    1762             $status = &delete_files (\@files);
    1763         }
    1764 
    1765         if ($status)  {
    1766             my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id";
    1767             if ($mode eq "goto_purged") {
    1768                 $command .= " -data_state purged";
    1769             }
    1770             elsif ($mode eq "goto_cleaned") {
    1771                 $command .= " -data_state cleaned";
    1772             }
    1773             elsif ($mode eq "goto_scrubbed") {
    1774                 $command .= " -data_state scrubbed";
    1775             }
    1776 
    1777             $command .= " -dbname $dbname" if defined $dbname;
    1778 
    1779             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1780                     run(command => $command, verbose => $verbose);
    1781             unless ($success) {
    1782                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1783                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1784             }
    1785         } else {
    1786             my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state";
    1787             $command .= " -dbname $dbname" if defined $dbname;
    1788 
    1789             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1790                     run(command => $command, verbose => $verbose);
    1791             unless ($success) {
    1792                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1793                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1794             }
    1795         }
    1796     }
    1797     # Check to see if we can mark the whole detRunSummary object as cleaned.
    1798 
    1799     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
    1800     $command .= " -dbname $dbname" if defined $dbname;
    1801     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    1802     unless ($success) {
    1803         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1804         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1805     }
    1806     if (@$stdout_buf != 0) {
    1807         $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1808             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1809         my $exps = parse_md_list($metadata) or
    1810             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1811        
    1812         foreach my $exp (@$exps) {
    1813             my $iteration = $exp->{iteration};
    1814             my $command;
    1815             if ($mode eq "goto_cleaned") {
    1816                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1817             }
    1818             if ($mode eq "goto_scrubbed") {
    1819                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1820             }
    1821             if ($mode eq "goto_purged") {
    1822                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1823             }
    1824             $command .= " -dbname $dbname" if defined $dbname;
    1825             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1826                 run(command => $command, verbose => $verbose);
    1827             unless ($success) {
    1828                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1829                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1830             }
    1831         }
    1832     }
    1833 
    1834     exit 0;
    1835 }
    1836 if ($stage eq "detrend.resid.exp") {
    1837     die "--stage_id required for stage $stage\n" if !$stage_id;
    1838     # this stage uses 'camtool'
    1839     my $dettool = can_run('dettool') or die "Can't find dettool";
    1840 
    1841     # Get list of component imfiles
    1842     # XXX may need a different my_die for each stage
    1843     my $exps;                      # Array of component files
    1844     my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run
    1845     $command .= " -dbname $dbname" if defined $dbname;
    1846     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    1847     unless ($success) {
    1848         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1849         &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1850     }
    1851     # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this
    1852     if (@$stdout_buf == 0) {
    1853         exit 0;
    1854     }
    1855     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1856         &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1857 
    1858     $exps = parse_md_list($metadata) or
    1859         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1860 
    1861     foreach my $exp (@$exps) {
    1862         my $exp_id = $exp->{exp_id};
    1863         my $iteration = $exp->{iteration};
    1864         my $path_base = $exp->{path_base};
    1865 #       my $class_id  = $exp->{class_id}
    1866         my $status = 1;
    1867         # don't clean up unless the data needed to update is available
    1868         # goto_scrubbed now requires the config file to not be present
    1869         if ($mode eq "goto_cleaned") {
    1870             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
    1871            
    1872             unless ($ipprc->file_exists($config_file)) {
    1873                 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
    1874                 $status = 0;
    1875             }
    1876         }
    1877         elsif ($mode eq "goto_scrubbed") {
    1878             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
    1879            
    1880             if ($ipprc->file_exists($config_file)) {
    1881                 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
    1882                 $status = 0;
    1883             }
    1884         }
    1885         if ($status) {
    1886             my @files = ();
    1887             # delete the temporary image datafiles
    1888             if ($mode eq "goto_purged") {
    1889                 # additional files to remove for 'purge' mode
    1890                 addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id);
    1891                 addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id);
    1892             }
    1893             # actual command to delete the files
    1894             $status = &delete_files (\@files);
    1895         }
    1896        
    1897         if ($status)  {
    1898             my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration";
    1899             if ($mode eq "goto_cleaned") {
    1900                 $command .= " -data_state cleaned";
    1901             }
    1902             if ($mode eq "goto_scrubbed") {
    1903                 $command .= " -data_state scrubbed";
    1904             }
    1905             if ($mode eq "goto_purged") {
    1906                 $command .= " -data_state purged";
    1907             }
    1908             $command .= " -dbname $dbname" if defined $dbname;
    1909             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1910                 run(command => $command, verbose => $verbose);
    1911             unless ($success) {
    1912                 print STDERR " residexp had an issue setting the state:? $success $error_code\n";
    1913                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1914                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1915             }
    1916         } else {
    1917             my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
    1918             $command .= " -dbname $dbname" if defined $dbname;
    1919            
    1920             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1921                 run(command => $command, verbose => $verbose);
    1922             unless ($success) {
    1923                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1924                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1925             }
    1926             exit $PS_EXIT_UNKNOWN_ERROR;
    1927         }
    1928     }
    1929     # Check to see if we can mark the whole detRunSummary object as cleaned.
    1930 
    1931     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
    1932     $command .= " -dbname $dbname" if defined $dbname;
    1933     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    1934     unless ($success) {
    1935         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1936         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1937     }
    1938     if (@$stdout_buf != 0) {
    1939         $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1940             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1941         $exps = parse_md_list($metadata) or
    1942             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1943        
    1944         foreach my $exp (@$exps) {
    1945             my $iteration = $exp->{iteration};
    1946             my $command;
    1947             if ($mode eq "goto_cleaned") {
    1948                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1949             }
    1950             if ($mode eq "goto_scrubbed") {
    1951                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1952             }
    1953             if ($mode eq "goto_purged") {
    1954                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1955             }
    1956             $command .= " -dbname $dbname" if defined $dbname;
    1957             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1958                 run(command => $command, verbose => $verbose);
    1959             unless ($success) {
    1960                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1961                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1962             }
    1963         }
    1964     }
    1965     exit 0;
    1966 }
    19671665
    19681666
Note: See TracChangeset for help on using the changeset viewer.