IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 24785


Ignore:
Timestamp:
Jul 13, 2009, 6:24:00 PM (17 years ago)
Author:
watersc1
Message:

Added more valid data_states to dettool.c

Fixed command line option comments for -data_state

ipp_cleanup.pl now contains preliminary handlers for the detrend cleanup.

Location:
branches/czw_branch/cleanup
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/cleanup/ippScripts/scripts/ipp_cleanup.pl

    r24743 r24785  
    747747
    748748}
    749 # fake : faketool : -pendingcleanupimfile (loop over imfiles)
     749# Detrend stages
     750if ($stage eq "detrend.process.imfile") {
     751
     752    die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id;
     753    ### select the imfiles for this entry
     754
     755    # this stage uses 'dettool'
     756    my $dettool = can_run('dettool') or die "Can't find chiptool";
     757
     758    # Get list of component imfiles
     759    # XXX may need a different my_die for each stage
     760    my $imfiles;                      # Array of component files
     761    my $command = "$dettool -pendingcleanup_processedimfile -chip_id $stage_id"; # Command to run
     762    $command .= " -dbname $dbname" if defined $dbname;
     763    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     764    unless ($success) {
     765        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     766        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
     767    }
     768
     769    # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
     770    if (@$stdout_buf == 0)  {
     771        my $command = "$dettool -det_id $stage_id -updateprocessedimfile -data_state new";
     772        $command .= " -dbname $dbname" if defined $dbname;
     773
     774        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     775            run(command => $command, verbose => $verbose);
     776        unless ($success) {
     777            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     778            &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     779        }
     780        exit 0;
     781    }
     782
     783    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     784        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     785
     786    # extract the metadata for the files into a hash list
     787    $imfiles = parse_md_list($metadata) or
     788        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     789
     790    # loop over all of the imfiles, determine the path_base and class_id for each
     791    foreach my $imfile (@$imfiles) {
     792        my $exp_id   = $imfile->{exp_id};
     793        my $class_id = $imfile->{class_id};
     794        my $path_base = $imfile->{path_base};
     795        my $status = 1;
     796
     797        # don't clean up unless the data needed to update is available
     798        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
     799        # goto_scrubbed now requires the config file to not exist.
     800       
     801        # Possibly not the correct config file, but simtest doesn't leave any around to check.
     802        if ($mode eq "goto_cleaned") {
     803            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     804
     805            if (!$config_file or ! -e $config_file) {
     806                print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id "
     807                    . " because config file is missing\n";
     808                $status = 0;
     809            }
     810        }
     811        elsif ($mode eq "goto_scrubbed") {
     812            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     813
     814            if ($config_file and -e $config_file) {
     815                print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id "
     816                    . " because config file is present\n";
     817                $status = 0;
     818            }
     819        }
     820
     821        if ($status) {
     822            # array of actual filenames to delete
     823            my @files = ();
     824
     825            # delete the temporary image datafiles
     826            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
     827            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
     828            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
     829            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
     830            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
     831            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
     832            if ($mode eq "goto_purged") {
     833                # additional files to remove for 'purge' mode
     834                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
     835                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
     836                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     837                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
     838                addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
     839                addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
     840                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
     841                addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
     842            }
     843
     844            # actual command to delete the files
     845            $status = &delete_files (\@files);
     846        }
     847
     848        if ($status)  {
     849            my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile";
     850            if ($mode eq "goto_purged") {
     851                $command .= " -data_state purged";
     852            }
     853            elsif ($mode eq "goto_cleaned") {
     854                $command .= " -data_state cleaned";
     855            }
     856            elsif ($mode eq "goto_scrubbed") {
     857                $command .= " -data_state scrubbed";
     858            }
     859
     860            $command .= " -dbname $dbname" if defined $dbname;
     861
     862            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     863                    run(command => $command, verbose => $verbose);
     864            unless ($success) {
     865                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     866                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     867            }
     868        } else {
     869
     870            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
     871            my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state";
     872            $command .= " -dbname $dbname" if defined $dbname;
     873
     874            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     875                    run(command => $command, verbose => $verbose);
     876            unless ($success) {
     877                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     878                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     879            }
     880        }
     881    }
     882    exit 0;
     883}
     884if ($stage eq "detrend.process.exp") {
     885    die "--stage_id required for stage $stage\n" if !$stage_id;
     886    # this stage uses 'camtool'
     887    my $camtool = can_run('dettool') or die "Can't find dettool";
     888
     889    # Get list of component imfiles
     890    # XXX may need a different my_die for each stage
     891    my $exps;                      # Array of component files
     892    my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run
     893    $command .= " -dbname $dbname" if defined $dbname;
     894    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     895    unless ($success) {
     896        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     897        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     898    }
     899    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     900        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     901
     902    $exps = parse_md_list($metadata) or
     903        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     904
     905
     906    foreach my $exp (@$exps) {
     907        my $path_base = $exp->{path_base};
     908        my $exp_id    = $exp->{exp_id};
     909
     910        my $status = 1;
     911        # don't clean up unless the data needed to update is available
     912        # goto_scrubbed now requires the config file to not be present
     913        if ($mode eq "goto_cleaned") {
     914            my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
     915           
     916            if (!$config_file or ! -e $config_file) {
     917                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
     918                $status = 0;
     919            }
     920        }
     921        elsif ($mode eq "goto_scrubbed") {
     922            my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
     923           
     924            if ($config_file and -e $config_file) {
     925                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
     926                $status = 0;
     927            }
     928        }
     929        if ($status) {
     930            my @files = ();
     931            # delete the temporary image datafiles
     932            addFilename (\@files, "PSASTRO.OUTPUT", $path_base);
     933            if ($mode eq "goto_purged") {
     934                # additional files to remove for 'purge' mode
     935                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
     936                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
     937                addFilename (\@files, "PSASTRO.STATS", $path_base);
     938            }
     939            # actual command to delete the files
     940            $status = &delete_files (\@files);
     941        }
     942       
     943        if ($status)  {
     944            my $command;
     945            if ($mode eq "goto_cleaned") {
     946                $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned";
     947            }
     948            if ($mode eq "goto_scrubbed") {
     949                $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed";
     950            }
     951            if ($mode eq "goto_purged") {
     952                $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged";
     953            }
     954            $command .= " -dbname $dbname" if defined $dbname;
     955            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     956                run(command => $command, verbose => $verbose);
     957            unless ($success) {
     958                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     959                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     960            }
     961        } else {
     962            # since 'camera' has only a single imfile, we can just update the run
     963            my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state";
     964            $command .= " -dbname $dbname" if defined $dbname;
     965           
     966            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     967                run(command => $command, verbose => $verbose);
     968            unless ($success) {
     969                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     970                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     971            }
     972            exit $PS_EXIT_UNKNOWN_ERROR;
     973        }
     974    }
     975    exit 0;
     976}
     977if ($stage eq "detrend.stack.imfile") {
     978   
     979    die "--stage_id required for stage $stage\n" if !$stage_id;
     980
     981    # this stage uses 'stacktool'
     982    my $stacktool = can_run('dettool') or die "Can't find dettool";
     983
     984    # Get list of component imfiles
     985    my $skyfiles;                  # Array reference of component files
     986    my $command = "dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run
     987    $command .= " -dbname $dbname" if defined $dbname;
     988    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     989    unless ($success) {
     990        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     991        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     992    }
     993    my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or
     994        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     995
     996    $skyfiles = parse_md_list($metadata) or
     997        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     998   
     999    my @files = ();
     1000    foreach my $stack (@{ $stacks }) {
     1001        # detStackedImfile does not have a path_base column.  This is inconvenient, as it means we need to calculate it.
     1002        my $path_base = $stack->{uri};
     1003        my $iteration = $stack->{iteration};
     1004        my $class_id  = $stack->{class_id};
     1005
     1006        $path_base =~ s/\.fits$//; # That should do it?
     1007
     1008        my $status = 1;
     1009        if ($mode eq "goto_cleaned") {
     1010            my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
     1011
     1012            $config_file =~ s%^file://%%;
     1013            if (!$config_file or ! -e $config_file) {
     1014                print STDERR "skipping cleanup for $stage $stage_id $path_base" .
     1015                    " because config file is missing\n";
     1016                $status = 0;
     1017            }
     1018            $config_file = 'file://' . $config_file;
     1019        }
     1020        elsif ($mode eq "goto_scrubbed") {
     1021            my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
     1022            $config_file =~ s%^file://%%;
     1023            if ($config_file and -e $config_file) {
     1024                print STDERR "skipping scrubbed for $stage $stage_id $path_base" .
     1025                    " because config file is present\n";
     1026                $status = 0;
     1027            }
     1028        }
     1029        if ($status) {
     1030            # delete the temporary image datafiles
     1031            addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $skycell_id);
     1032            addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $skycell_id);
     1033            addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $skycell_id);
     1034
     1035            if ($mode eq "goto_purged") {
     1036                # additional files to remove for 'purge' mode
     1037                # Nothing?
     1038            }
     1039
     1040            $status = &delete_files(\@files);
     1041        }
     1042
     1043        if ($status) {
     1044            my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id";
     1045            if ($mode eq "goto_purged") {
     1046                $command .= " -updatestacked -data_state purged";
     1047            }
     1048            elsif ($mode eq "goto_cleaned") {
     1049                $command .= " -updatestacked -data_state cleaned";
     1050            }
     1051            elsif ($mode eq "goto_scrubbed") {
     1052                $command .= " -updatestacked -data_state scrubbed";
     1053            }
     1054            $command .= " -dbname $dbname" if defined $dbname;
     1055           
     1056            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1057                run(command => $command, verbose => $verbose);
     1058            unless ($success) {
     1059                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1060                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1061            }
     1062        } else {
     1063            my $command = "$dettool -updatestacked  -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1064            $command .= " -dbname $dbname" if defined $dbname;
     1065           
     1066            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1067                run(command => $command, verbose => $verbose);
     1068            unless ($success) {
     1069                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1070                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1071            }
     1072            exit $PS_EXIT_UNKNOWN_ERROR;
     1073        }
     1074    }
     1075    exit 0;
     1076}
     1077if ($stage eq "detrend.normstat.imfile") {
     1078    print STDERR "I'm not convinced there's anything to clean up from stage $stage\n";
     1079    die "--stage_id required for stage $stage\n" if !$stage_id;
     1080    # this stage uses 'camtool'
     1081    my $camtool = can_run('dettool') or die "Can't find dettool";
     1082
     1083    my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run
     1084    $command .= " -dbname $dbname" if defined $dbname;
     1085    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1086    unless ($success) {
     1087        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1088        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1089    }
     1090    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1091        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1092
     1093    $exps = parse_md_list($metadata) or
     1094        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1095
     1096    foreach my $exp (@$exps) {
     1097        my $path_base = $exp->{path_base};
     1098        my $iteration = $exp->{iteration};
     1099        my $class_id  = $exp->{class_id};
     1100
     1101        my $status = 1;
     1102        if ($status)  {
     1103            my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id";
     1104            if ($mode eq "goto_cleaned") {
     1105                $command .= " -data_state cleaned";
     1106            }
     1107            if ($mode eq "goto_scrubbed") {
     1108                $command .= " -data_state scrubbed";
     1109            }
     1110            if ($mode eq "goto_purged") {
     1111                $command .= " -data_state purged";
     1112            }
     1113            $command .= " -dbname $dbname" if defined $dbname;
     1114            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1115                run(command => $command, verbose => $verbose);
     1116            unless ($success) {
     1117                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1118                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1119            }
     1120        } else {
     1121            # since 'camera' has only a single imfile, we can just update the run
     1122            my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1123            $command .= " -dbname $dbname" if defined $dbname;
     1124           
     1125            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1126                run(command => $command, verbose => $verbose);
     1127            unless ($success) {
     1128                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1129                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1130            }
     1131            exit $PS_EXIT_UNKNOWN_ERROR;
     1132        }
     1133    }
     1134    exit 0;
     1135
     1136}
     1137if ($stage eq "detrend.norm.imfile") {
     1138    die "--stage_id required for stage $stage\n" if !$stage_id;
     1139    # this stage uses 'camtool'
     1140    my $camtool = can_run('dettool') or die "Can't find dettool";
     1141
     1142    # Get list of component imfiles
     1143    # XXX may need a different my_die for each stage
     1144    my $exps;                      # Array of component files
     1145    my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run
     1146    $command .= " -dbname $dbname" if defined $dbname;
     1147    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1148    unless ($success) {
     1149        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1150        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1151    }
     1152    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1153        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1154
     1155    $exps = parse_md_list($metadata) or
     1156        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1157
     1158    foreach my $exp (@$exps) {
     1159        my $path_base = $exp->{path_base};
     1160        my $iteration = $exp->{iteration};
     1161        my $class_id  = $exp->{class_id};
     1162
     1163        my $status = 1;
     1164        # don't clean up unless the data needed to update is available
     1165        # goto_scrubbed now requires the config file to not be present
     1166        if ($mode eq "goto_cleaned") {
     1167            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1168           
     1169            if (!$config_file or ! -e $config_file) {
     1170                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
     1171                $status = 0;
     1172            }
     1173        }
     1174        elsif ($mode eq "goto_scrubbed") {
     1175            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1176           
     1177            if ($config_file and -e $config_file) {
     1178                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
     1179                $status = 0;
     1180            }
     1181        }
     1182        if ($status) {
     1183            my @files = ();
     1184            # delete the temporary image datafiles
     1185            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base);
     1186            addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base);
     1187            addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base);
     1188            if ($mode eq "goto_purged") {
     1189                # additional files to remove for 'purge' mode
     1190                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
     1191                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
     1192            }
     1193            # actual command to delete the files
     1194            $status = &delete_files (\@files);
     1195        }
     1196       
     1197        if ($status)  {
     1198            my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id";
     1199            if ($mode eq "goto_cleaned") {
     1200                $command .= " -data_state cleaned";
     1201            }
     1202            if ($mode eq "goto_scrubbed") {
     1203                $command .= " -data_state scrubbed";
     1204            }
     1205            if ($mode eq "goto_purged") {
     1206                $command .= " -data_state purged";
     1207            }
     1208            $command .= " -dbname $dbname" if defined $dbname;
     1209            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1210                run(command => $command, verbose => $verbose);
     1211            unless ($success) {
     1212                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1213                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1214            }
     1215        } else {
     1216            my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1217            $command .= " -dbname $dbname" if defined $dbname;
     1218           
     1219            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1220                run(command => $command, verbose => $verbose);
     1221            unless ($success) {
     1222                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1223                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1224            }
     1225            exit $PS_EXIT_UNKNOWN_ERROR;
     1226        }
     1227    }
     1228    exit 0;
     1229}
     1230if ($stage eq "detrend.resid.imfile") {
     1231
     1232    die "--stage_id required for stage $stage\n" if !$stage_id;
     1233    ### select the imfiles for this entry
     1234
     1235    # this stage uses 'dettool'
     1236    my $dettool = can_run('dettool') or die "Can't find dettool";
     1237
     1238    # Get list of component imfiles
     1239    # XXX may need a different my_die for each stage
     1240    my $imfiles;                      # Array of component files
     1241    my $command = "$dettool -pendingcleanup_residimfile -chip_id $stage_id"; # Command to run
     1242    $command .= " -dbname $dbname" if defined $dbname;
     1243    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1244    unless ($success) {
     1245        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1246        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
     1247    }
     1248
     1249    # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
     1250    if (@$stdout_buf == 0)  {
     1251        my $command = "$dettool -det_id $stage_id -updateresidimfile -data_state new";
     1252        $command .= " -dbname $dbname" if defined $dbname;
     1253
     1254        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1255            run(command => $command, verbose => $verbose);
     1256        unless ($success) {
     1257            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1258            &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1259        }
     1260        exit 0;
     1261    }
     1262
     1263    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1264        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1265
     1266    # extract the metadata for the files into a hash list
     1267    $imfiles = parse_md_list($metadata) or
     1268        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1269
     1270    # loop over all of the imfiles, determine the path_base and class_id for each
     1271    foreach my $imfile (@$imfiles) {
     1272        my $class_id = $imfile->{class_id};
     1273        my $iteration = $imfile->{iteration};
     1274        my $exp_id = $imfile->{exp_id};
     1275        my $path_base = $imfile->{path_base};
     1276        my $status = 1;
     1277
     1278        # don't clean up unless the data needed to update is available
     1279        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
     1280        # goto_scrubbed now requires the config file to not exist.
     1281       
     1282        # Possibly not the correct config file, but simtest doesn't leave any around to check.
     1283        if ($mode eq "goto_cleaned") {
     1284            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     1285
     1286            if (!$config_file or ! -e $config_file) {
     1287                print STDERR "skipping cleanup for $stage $stage_id $class_id "
     1288                    . " because config file is missing\n";
     1289                $status = 0;
     1290            }
     1291        }
     1292        elsif ($mode eq "goto_scrubbed") {
     1293            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     1294
     1295            if ($config_file and -e $config_file) {
     1296                print STDERR "skipping scrubbed for $stage $stage_id $class_id "
     1297                    . " because config file is present\n";
     1298                $status = 0;
     1299            }
     1300        }
     1301
     1302        if ($status) {
     1303            # array of actual filenames to delete
     1304            my @files = ();
     1305
     1306            # delete the temporary image datafiles
     1307            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
     1308            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
     1309            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
     1310            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
     1311            if ($mode eq "goto_purged") {
     1312                # additional files to remove for 'purge' mode
     1313                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
     1314                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
     1315                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     1316                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
     1317                addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
     1318                addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
     1319                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
     1320                addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
     1321            }
     1322
     1323            # actual command to delete the files
     1324            $status = &delete_files (\@files);
     1325        }
     1326
     1327        if ($status)  {
     1328            my $command = "$dettool -updateresidimfile -chip_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id";
     1329            if ($mode eq "goto_purged") {
     1330                $command .= " -data_state purged";
     1331            }
     1332            elsif ($mode eq "goto_cleaned") {
     1333                $command .= " -data_state cleaned";
     1334            }
     1335            elsif ($mode eq "goto_scrubbed") {
     1336                $command .= " -data_state scrubbed";
     1337            }
     1338
     1339            $command .= " -dbname $dbname" if defined $dbname;
     1340
     1341            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1342                    run(command => $command, verbose => $verbose);
     1343            unless ($success) {
     1344                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1345                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1346            }
     1347        } else {
     1348            my $command = "$dettool -updateresidimfile -chip_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1349            $command .= " -dbname $dbname" if defined $dbname;
     1350
     1351            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1352                    run(command => $command, verbose => $verbose);
     1353            unless ($success) {
     1354                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1355                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1356            }
     1357        }
     1358    }
     1359    exit 0;
     1360}
     1361if ($stage eq "detrend.resid.exp") {
     1362    die "--stage_id required for stage $stage\n" if !$stage_id;
     1363    # this stage uses 'camtool'
     1364    my $camtool = can_run('dettool') or die "Can't find dettool";
     1365
     1366    # Get list of component imfiles
     1367    # XXX may need a different my_die for each stage
     1368    my $exps;                      # Array of component files
     1369    my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run
     1370    $command .= " -dbname $dbname" if defined $dbname;
     1371    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1372    unless ($success) {
     1373        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1374        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1375    }
     1376    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1377        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1378
     1379    $exps = parse_md_list($metadata) or
     1380        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1381
     1382    foreach my $exp (@$exps) {
     1383        my $exp_id = $exp->{exp_id};
     1384        my $iteration = $exp->{iteration};
     1385        my $path_base = $exp->{path_base};
     1386
     1387        my $status = 1;
     1388        # don't clean up unless the data needed to update is available
     1389        # goto_scrubbed now requires the config file to not be present
     1390        if ($mode eq "goto_cleaned") {
     1391            my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
     1392           
     1393            if (!$config_file or ! -e $config_file) {
     1394                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
     1395                $status = 0;
     1396            }
     1397        }
     1398        elsif ($mode eq "goto_scrubbed") {
     1399            my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
     1400           
     1401            if ($config_file and -e $config_file) {
     1402                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
     1403                $status = 0;
     1404            }
     1405        }
     1406        if ($status) {
     1407            my @files = ();
     1408            # delete the temporary image datafiles
     1409            addFilename (\@files, "PSASTRO.OUTPUT", $path_base);
     1410            if ($mode eq "goto_purged") {
     1411                # additional files to remove for 'purge' mode
     1412                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
     1413                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
     1414                addFilename (\@files, "PSASTRO.STATS", $path_base);
     1415            }
     1416            # actual command to delete the files
     1417            $status = &delete_files (\@files);
     1418        }
     1419       
     1420        if ($status)  {
     1421            my $command = "$dettool -updateresiddexp -det_id $stage_id -exp_id $exp_id -iteration $iteration";
     1422            if ($mode eq "goto_cleaned") {
     1423                $command .= " -data_state cleaned";
     1424            }
     1425            if ($mode eq "goto_scrubbed") {
     1426                $command .= " -data_state scrubbed";
     1427            }
     1428            if ($mode eq "goto_purged") {
     1429                $command .= " -data_state purged";
     1430            }
     1431            $command .= " -dbname $dbname" if defined $dbname;
     1432            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1433                run(command => $command, verbose => $verbose);
     1434            unless ($success) {
     1435                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1436                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1437            }
     1438        } else {
     1439            my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
     1440            $command .= " -dbname $dbname" if defined $dbname;
     1441           
     1442            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1443                run(command => $command, verbose => $verbose);
     1444            unless ($success) {
     1445                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1446                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1447            }
     1448            exit $PS_EXIT_UNKNOWN_ERROR;
     1449        }
     1450    }
     1451    exit 0;
     1452}
     1453
    7501454
    7511455die "ipp_cleanup.pl -stage $stage not yet implemented\n";

  • branches/czw_branch/cleanup/ippTools/src/dettool.c

    r23829 r24785  
    18171817    if (!strcmp(data_state, "drop")) return true;
    18181818    if (!strcmp(data_state, "register")) return true;
     1819    // These are valid data states, and are necessary for the cleanup to work correctly.
     1820    if (!strcmp(data_state, "full")) return true;
     1821    if (!strcmp(data_state, "goto_cleaned")) return true;
     1822    if (!strcmp(data_state, "goto_scrubbed")) return true;
     1823    if (!strcmp(data_state, "goto_purged")) return true;
     1824    if (!strcmp(data_state, "cleaned")) return true;
     1825    if (!strcmp(data_state, "scrubbed")) return true;
     1826    if (!strcmp(data_state, "purged")) return true;
     1827    if (!strcmp(data_state, "error_cleaned")) return true;
     1828    if (!strcmp(data_state, "error_scrubbed")) return true;
     1829    if (!strcmp(data_state, "error_purged")) return true;
    18191830
    18201831    psError(PS_ERR_UNKNOWN, true, "invalid data state: %s", data_state);

  • branches/czw_branch/cleanup/ippTools/src/dettoolConfig.c

    r23688 r24785  
    4141    // XXX EAM : is this used?  does it make sense?
    4242    psMetadata *pendingArgs = psMetadataAlloc();
    43     psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exposure ID", 0);
    44     psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type",  0,            "search by exposure type", NULL);
    45     psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst",  0,            "search by camera", NULL);
    46     psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope",  0,            "search by telescope", NULL);
    47     psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter",  0,            "search by filter", NULL);
     43    psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id",  0,         "search by exposure ID", 0);
     44    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type",  0,       "search by exposure type", NULL);
     45    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst",  0,           "search by camera", NULL);
     46    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope",  0,      "search by telescope", NULL);
     47    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter",  0,         "search by filter", NULL);
    4848    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri",  0,            "search by URL", NULL);
    49     psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     49    psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple",  0,        "use the simple output format", false);
    5050
    5151    // -definebytag
    … …  
    195195    // -addcorrectimfile
    196196    psMetadata *addcorrectimfileArgs = psMetadataAlloc();
    197     psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id",  0,            "define detrend ID (required)", 0);
    198     psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id",  0,            "search for class ID (required)", NULL);
    199     psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri",  0,            "define resid file URI", NULL);
     197    psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id",  0,        "define detrend ID (required)", 0);
     198    psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id",  0,      "search for class ID (required)", NULL);
     199    psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri",  0,           "define resid file URI", NULL);
    200200    psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg",  0,            "define exposure background", NAN);
    201201    psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_stdev",  0,            "define exposure background stdev", NAN);
    … …  
    294294    psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id",               0,            "search by exp_id", 0);
    295295    psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
    296     psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
     296    psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
    297297
    298298    // -pendingcleanup_processedimfile
    … …  
    354354    psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id",               0,            "search by chip ID", 0);
    355355    psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id",               0,            "search by exp_id", 0);
    356     psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
     356    psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
    357357
    358358    // -pendingcleanup_processedexp
    … …  
    414414    psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration",  0,            "search by iteration number", 0);
    415415    psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
    416     psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
     416    psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
    417417
    418418    // -pendingcleanup_stacked
    … …  
    466466    psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration",  0,            "search by iteration number", 0);
    467467    psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
    468     psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
     468    psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
    469469
    470470    // -pendingcleanup_normalizedstat
    … …  
    528528    psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
    529529    psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
    530     psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
     530    psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
    531531
    532532    // -pendingcleanup_normalizedimfile
    … …  
    586586    psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id",               0,            "search by chip ID", 0);
    587587    psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
    588     psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
     588    psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
    589589
    590590    // -pendingcleanup_normalizedexp
    … …  
    666666    psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id",               0,            "search by exp_id", 0);
    667667    psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
    668     psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
     668    psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
    669669
    670670    // -pendingcleanup_residimfile
    … …  
    747747    psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base",  0,            "define base output location", NULL);
    748748    psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject",  0,            "exposure is not to be stacked in the next iteration", false);
    749     psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
     749    psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
    750750
    751751    // -pendingcleanup_residexp
Note: See TracChangeset for help on using the changeset viewer.