IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jul 31, 2009, 6:33:58 PM (17 years ago)
Author:
watersc1
Message:

Detrend cleanup. It works on simtest data. Finally.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/cleanup/ippScripts/scripts/ipp_cleanup.pl

    r24951 r24972  
    767767    }
    768768
    769     # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
    770     if (@$stdout_buf == 0)  {
    771         my $command = "$dettool -det_id $stage_id -updateprocessedimfile -data_state new";
    772         $command .= " -dbname $dbname" if defined $dbname;
    773 
    774         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    775             run(command => $command, verbose => $verbose);
    776         unless ($success) {
    777             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    778             &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    779         }
    780         exit 0;
     769#     # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
     770#     if (@$stdout_buf == 0)  {
     771#       my $command = "$dettool -det_id $stage_id -updateprocessedimfile -data_state new";
     772#       $command .= " -dbname $dbname" if defined $dbname;
     773
     774#       my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     775#           run(command => $command, verbose => $verbose);
     776#       unless ($success) {
     777#           $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     778#           &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     779#       }
     780#       exit 0;
     781#     }
     782   
     783    if (@$stdout_buf == 0) {
     784        exit 0; #silently abort. I need to fix this for propers
    781785    }
    782786
    … …  
    891895        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    892896    }
    893     $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    894         &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    895     $exps = parse_md_list($metadata) or
    896         &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    897 
    898     foreach my $exp (@$exps) {
    899         my $iteration = $exp->{iteration};
    900         my $command;
    901         if ($mode eq "goto_cleaned") {
    902             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    903         }
    904         if ($mode eq "goto_scrubbed") {
    905             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    906         }
    907         if ($mode eq "goto_purged") {
    908             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    909         }
    910         $command .= " -dbname $dbname" if defined $dbname;
    911         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    912             run(command => $command, verbose => $verbose);
    913         unless ($success) {
    914             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    915             &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    916         }
     897    if (@$stdout_buf != 0) {
     898        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     899            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     900        my $exps = parse_md_list($metadata) or
     901            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     902       
     903        foreach my $exp (@$exps) {
     904            my $iteration = $exp->{iteration};
     905            my $command;
     906            if ($mode eq "goto_cleaned") {
     907                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     908            }
     909            if ($mode eq "goto_scrubbed") {
     910                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     911            }
     912            if ($mode eq "goto_purged") {
     913                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     914            }
     915            $command .= " -dbname $dbname" if defined $dbname;
     916            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     917                run(command => $command, verbose => $verbose);
     918            unless ($success) {
     919                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     920                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     921            }
     922        }
    917923    }
    918924   
    … …  
    921927if ($stage eq "detrend.process.exp") {
    922928    die "--stage_id required for stage $stage\n" if !$stage_id;
    923     # this stage uses 'camtool'
    924     my $camtool = can_run('dettool') or die "Can't find dettool";
     929    # this stage uses 'dettool'
     930    my $dettool = can_run('dettool') or die "Can't find dettool";
    925931
    926932    # Get list of component imfiles
    … …  
    934940        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    935941    }
     942   
     943    if (@$stdout_buf == 0) {
     944        exit 0; #silently abort. I need to fix this for propers
     945    }
     946
    936947    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    937948        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    … …  
    10191030        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    10201031    }
    1021     $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1022         &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1023     $exps = parse_md_list($metadata) or
    1024         &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1025 
    1026     foreach my $exp (@$exps) {
    1027         my $iteration = $exp->{iteration};
    1028         my $command;
    1029         if ($mode eq "goto_cleaned") {
    1030             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1031         }
    1032         if ($mode eq "goto_scrubbed") {
    1033             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1034         }
    1035         if ($mode eq "goto_purged") {
    1036             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1037         }
    1038         $command .= " -dbname $dbname" if defined $dbname;
    1039         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1040             run(command => $command, verbose => $verbose);
    1041         unless ($success) {
    1042             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1043             &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1044         }
     1032    if (@$stdout_buf != 0) {
     1033        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1034            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1035        $exps = parse_md_list($metadata) or
     1036            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1037       
     1038        foreach my $exp (@$exps) {
     1039            my $iteration = $exp->{iteration};
     1040            my $command;
     1041            if ($mode eq "goto_cleaned") {
     1042                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1043            }
     1044            if ($mode eq "goto_scrubbed") {
     1045                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1046            }
     1047            if ($mode eq "goto_purged") {
     1048                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1049            }
     1050            $command .= " -dbname $dbname" if defined $dbname;
     1051            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1052                run(command => $command, verbose => $verbose);
     1053            unless ($success) {
     1054                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1055                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1056            }
     1057        }
    10451058    }
    10461059
    … …  
    10511064    die "--stage_id required for stage $stage\n" if !$stage_id;
    10521065
    1053     # this stage uses 'stacktool'
    1054     my $stacktool = can_run('dettool') or die "Can't find dettool";
     1066    # this stage uses 'dettool'
     1067    my $dettool = can_run('dettool') or die "Can't find dettool";
    10551068
    10561069    # Get list of component imfiles
    1057     my $skyfiles;                  # Array reference of component files
    1058     my $command = "dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run
     1070    my $stacks;                  # Array reference of component files
     1071    my $command = "$dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run
    10591072    $command .= " -dbname $dbname" if defined $dbname;
    10601073    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    … …  
    10661079        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    10671080
    1068     $skyfiles = parse_md_list($metadata) or
     1081    $stacks = parse_md_list($metadata) or
    10691082        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    10701083   
    … …  
    10791092
    10801093        my $status = 1;
    1081         if ($mode eq "goto_cleaned") {
    1082             my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
    1083 
    1084             $config_file =~ s%^file://%%;
    1085             if (!$config_file or ! -e $config_file) {
    1086                 print STDERR "skipping cleanup for $stage $stage_id $path_base" .
    1087                     " because config file is missing\n";
    1088                 $status = 0;
    1089             }
    1090             $config_file = 'file://' . $config_file;
    1091         }
    1092         elsif ($mode eq "goto_scrubbed") {
    1093             my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
    1094             $config_file =~ s%^file://%%;
    1095             if ($config_file and -e $config_file) {
    1096                 print STDERR "skipping scrubbed for $stage $stage_id $path_base" .
    1097                     " because config file is present\n";
    1098                 $status = 0;
    1099             }
    1100         }
     1094#       if ($mode eq "goto_cleaned") {
     1095#           my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
     1096
     1097#           $config_file =~ s%^file://%%;
     1098#           if (!$config_file or ! -e $config_file) {
     1099#               print STDERR "skipping cleanup for $stage $stage_id $path_base" .
     1100#                   " because config file is missing\n";
     1101#               $status = 0;
     1102#           }
     1103#           $config_file = 'file://' . $config_file;
     1104#       }
     1105#       elsif ($mode eq "goto_scrubbed") {
     1106#           my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
     1107#           $config_file =~ s%^file://%%;
     1108#           if ($config_file and -e $config_file) {
     1109#               print STDERR "skipping scrubbed for $stage $stage_id $path_base" .
     1110#                   " because config file is present\n";
     1111#               $status = 0;
     1112#           }
     1113#       }
    11011114        if ($status) {
    11021115            # delete the temporary image datafiles
    1103             addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $skycell_id);
    1104             addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $skycell_id);
    1105             addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $skycell_id);
     1116#           addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id);
     1117            addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id);
     1118            addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id);
    11061119
    11071120            if ($mode eq "goto_purged") {
    … …  
    11541167        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    11551168    }
    1156     $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1157         &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1158     $exps = parse_md_list($metadata) or
    1159         &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1160 
    1161     foreach my $exp (@$exps) {
    1162         my $iteration = $exp->{iteration};
    1163         my $command;
    1164         if ($mode eq "goto_cleaned") {
    1165             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1166         }
    1167         if ($mode eq "goto_scrubbed") {
    1168             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1169         }
    1170         if ($mode eq "goto_purged") {
    1171             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1172         }
    1173         $command .= " -dbname $dbname" if defined $dbname;
    1174         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1175             run(command => $command, verbose => $verbose);
    1176         unless ($success) {
    1177             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1178             &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1179         }
    1180     }
    1181 
     1169    if (@$stdout_buf != 0) {
     1170        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1171            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1172        my $exps = parse_md_list($metadata) or
     1173            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1174       
     1175        foreach my $exp (@$exps) {
     1176            my $iteration = $exp->{iteration};
     1177            my $command;
     1178            if ($mode eq "goto_cleaned") {
     1179                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1180            }
     1181            if ($mode eq "goto_scrubbed") {
     1182                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1183            }
     1184            if ($mode eq "goto_purged") {
     1185                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1186            }
     1187            $command .= " -dbname $dbname" if defined $dbname;
     1188            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1189                run(command => $command, verbose => $verbose);
     1190            unless ($success) {
     1191                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1192                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1193            }
     1194        }
     1195    }
    11821196    exit 0;
    11831197}
    … …  
    11861200    die "--stage_id required for stage $stage\n" if !$stage_id;
    11871201    # this stage uses 'camtool'
    1188     my $camtool = can_run('dettool') or die "Can't find dettool";
     1202    my $dettool = can_run('dettool') or die "Can't find dettool";
    11891203
    11901204    my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run
    … …  
    11981212        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    11991213
    1200     $exps = parse_md_list($metadata) or
     1214    my $exps = parse_md_list($metadata) or
    12011215        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    12021216
    12031217    foreach my $exp (@$exps) {
    1204         my $path_base = $exp->{path_base};
     1218#       my $path_base = $exp->{path_base};
    12051219        my $iteration = $exp->{iteration};
    12061220        my $class_id  = $exp->{class_id};
    … …  
    12221236                run(command => $command, verbose => $verbose);
    12231237            unless ($success) {
     1238                print STDERR "WTF: $success TTT $error_code QQQ\n";
    12241239                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    12251240                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    … …  
    12481263        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    12491264    }
    1250     $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1251         &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1252     $exps = parse_md_list($metadata) or
    1253         &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1254 
    1255     foreach my $exp (@$exps) {
    1256         my $iteration = $exp->{iteration};
    1257         my $command;
    1258         if ($mode eq "goto_cleaned") {
    1259             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1260         }
    1261         if ($mode eq "goto_scrubbed") {
    1262             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1263         }
    1264         if ($mode eq "goto_purged") {
    1265             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1266         }
    1267         $command .= " -dbname $dbname" if defined $dbname;
    1268         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1269             run(command => $command, verbose => $verbose);
    1270         unless ($success) {
    1271             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1272             &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1273         }
     1265    if (@$stdout_buf != 0) {
     1266        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1267            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1268        $exps = parse_md_list($metadata) or
     1269            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1270       
     1271        foreach my $exp (@$exps) {
     1272            my $iteration = $exp->{iteration};
     1273            my $command;
     1274            if ($mode eq "goto_cleaned") {
     1275                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1276            }
     1277            if ($mode eq "goto_scrubbed") {
     1278                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1279            }
     1280            if ($mode eq "goto_purged") {
     1281                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1282            }
     1283            $command .= " -dbname $dbname" if defined $dbname;
     1284            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1285                run(command => $command, verbose => $verbose);
     1286            unless ($success) {
     1287                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1288                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1289            }
     1290        }
    12741291    }
    12751292
    … …  
    12801297    die "--stage_id required for stage $stage\n" if !$stage_id;
    12811298    # this stage uses 'camtool'
    1282     my $camtool = can_run('dettool') or die "Can't find dettool";
     1299    my $dettool = can_run('dettool') or die "Can't find dettool";
    12831300
    12841301    # Get list of component imfiles
    … …  
    13771394        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    13781395    }
    1379     $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1380         &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1396    if (@$stdout_buf != 0) {
     1397        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1398            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1399        $exps = parse_md_list($metadata) or
     1400            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1401       
     1402        foreach my $exp (@$exps) {
     1403            my $iteration = $exp->{iteration};
     1404            my $command;
     1405            if ($mode eq "goto_cleaned") {
     1406                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1407            }
     1408            if ($mode eq "goto_scrubbed") {
     1409                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1410            }
     1411            if ($mode eq "goto_purged") {
     1412                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1413            }
     1414            $command .= " -dbname $dbname" if defined $dbname;
     1415            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1416                run(command => $command, verbose => $verbose);
     1417            unless ($success) {
     1418                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1419                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1420            }
     1421        }
     1422    }
     1423
     1424    exit 0;
     1425}
     1426if ($stage eq "detrend.norm.exp") {
     1427    die "--stage_id required for stage $stage\n" if !$stage_id;
     1428    # this stage uses 'camtool'
     1429    my $dettool = can_run('dettool') or die "Can't find dettool";
     1430
     1431    # Get list of component imfiles
     1432    # XXX may need a different my_die for each stage
     1433    my $exps;                      # Array of component files
     1434    my $command = "$dettool -pendingcleanup_normalizedexp -det_id $stage_id"; # Command to run
     1435    $command .= " -dbname $dbname" if defined $dbname;
     1436    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1437    unless ($success) {
     1438        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1439        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1440    }
     1441    # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this
     1442    if (@$stdout_buf == 0) {
     1443        exit 0;
     1444    }
     1445    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1446        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1447
    13811448    $exps = parse_md_list($metadata) or
    1382         &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1449        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    13831450
    13841451    foreach my $exp (@$exps) {
     1452        my $exp_id = $exp->{exp_id};
    13851453        my $iteration = $exp->{iteration};
    1386         my $command;
     1454        my $path_base = $exp->{path_base};
     1455
     1456        my $status = 1;
     1457        # don't clean up unless the data needed to update is available
     1458        # goto_scrubbed now requires the config file to not be present
    13871459        if ($mode eq "goto_cleaned") {
    1388             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1389         }
    1390         if ($mode eq "goto_scrubbed") {
    1391             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1392         }
    1393         if ($mode eq "goto_purged") {
    1394             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1395         }
    1396         $command .= " -dbname $dbname" if defined $dbname;
    1397         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1398             run(command => $command, verbose => $verbose);
    1399         unless ($success) {
    1400             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1401             &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1402         }
    1403     }
    1404 
     1460            my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
     1461           
     1462            if (!$config_file or ! -e $config_file) {
     1463                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
     1464                $status = 0;
     1465            }
     1466        }
     1467        elsif ($mode eq "goto_scrubbed") {
     1468            my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
     1469           
     1470            if ($config_file and -e $config_file) {
     1471                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
     1472                $status = 0;
     1473            }
     1474        }
     1475        if ($status) {
     1476            my @files = ();
     1477            # delete the temporary image datafiles
     1478#           addFilename (\@files, "PSASTRO.OUTPUT", $path_base);
     1479            if ($mode eq "goto_purged") {
     1480                # additional files to remove for 'purge' mode
     1481                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
     1482                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
     1483#               addFilename (\@files, "PSASTRO.STATS", $path_base);
     1484            }
     1485            # actual command to delete the files
     1486            $status = &delete_files (\@files);
     1487        }
     1488       
     1489        if ($status)  {
     1490            my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration";
     1491            if ($mode eq "goto_cleaned") {
     1492                $command .= " -data_state cleaned";
     1493            }
     1494            if ($mode eq "goto_scrubbed") {
     1495                $command .= " -data_state scrubbed";
     1496            }
     1497            if ($mode eq "goto_purged") {
     1498                $command .= " -data_state purged";
     1499            }
     1500            $command .= " -dbname $dbname" if defined $dbname;
     1501            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1502                run(command => $command, verbose => $verbose);
     1503            unless ($success) {
     1504                print STDERR " residexp had an issue setting the state:? $success $error_code\n";
     1505                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1506                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1507            }
     1508        } else {
     1509            my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
     1510            $command .= " -dbname $dbname" if defined $dbname;
     1511           
     1512            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1513                run(command => $command, verbose => $verbose);
     1514            unless ($success) {
     1515                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1516                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1517            }
     1518            exit $PS_EXIT_UNKNOWN_ERROR;
     1519        }
     1520    }
     1521    # Check to see if we can mark the whole detRunSummary object as cleaned.
     1522
     1523    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     1524    $command .= " -dbname $dbname" if defined $dbname;
     1525    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     1526    unless ($success) {
     1527        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1528        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1529    }
     1530    if (@$stdout_buf != 0) {
     1531        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1532            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1533        $exps = parse_md_list($metadata) or
     1534            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1535       
     1536        foreach my $exp (@$exps) {
     1537            my $iteration = $exp->{iteration};
     1538            my $command;
     1539            if ($mode eq "goto_cleaned") {
     1540                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1541            }
     1542            if ($mode eq "goto_scrubbed") {
     1543                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1544            }
     1545            if ($mode eq "goto_purged") {
     1546                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1547            }
     1548            $command .= " -dbname $dbname" if defined $dbname;
     1549            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1550                run(command => $command, verbose => $verbose);
     1551            unless ($success) {
     1552                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1553                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1554            }
     1555        }
     1556    }
    14051557    exit 0;
    14061558}
    … …  
    14251577
    14261578    # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
    1427     if (@$stdout_buf == 0)  {
    1428         my $command = "$dettool -det_id $stage_id -updateresidimfile -data_state new";
    1429         $command .= " -dbname $dbname" if defined $dbname;
    1430 
    1431         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1432             run(command => $command, verbose => $verbose);
    1433         unless ($success) {
    1434             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1435             &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1436         }
    1437         exit 0;
     1579#     if (@$stdout_buf == 0)  {
     1580#       my $command = "$dettool -det_id $stage_id -updateresidimfile -data_state new";
     1581#       $command .= " -dbname $dbname" if defined $dbname;
     1582
     1583#       my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1584#           run(command => $command, verbose => $verbose);
     1585#       unless ($success) {
     1586#           $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1587#           &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1588#       }
     1589#       exit 0;
     1590#     }
     1591   
     1592    if (@$stdout_buf == 0) {
     1593        exit 0; #silently abort. I need to fix this for propers
    14381594    }
    14391595
    … …  
    15431699        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    15441700    }
    1545     $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1546         &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1547     $exps = parse_md_list($metadata) or
    1548         &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1549 
    1550     foreach my $exp (@$exps) {
    1551         my $iteration = $exp->{iteration};
    1552         my $command;
    1553         if ($mode eq "goto_cleaned") {
    1554             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1555         }
    1556         if ($mode eq "goto_scrubbed") {
    1557             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1558         }
    1559         if ($mode eq "goto_purged") {
    1560             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1561         }
    1562         $command .= " -dbname $dbname" if defined $dbname;
    1563         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1564             run(command => $command, verbose => $verbose);
    1565         unless ($success) {
    1566             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1567             &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1568         }
     1701    if (@$stdout_buf != 0) {
     1702        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1703            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1704        my $exps = parse_md_list($metadata) or
     1705            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1706       
     1707        foreach my $exp (@$exps) {
     1708            my $iteration = $exp->{iteration};
     1709            my $command;
     1710            if ($mode eq "goto_cleaned") {
     1711                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1712            }
     1713            if ($mode eq "goto_scrubbed") {
     1714                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1715            }
     1716            if ($mode eq "goto_purged") {
     1717                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1718            }
     1719            $command .= " -dbname $dbname" if defined $dbname;
     1720            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1721                run(command => $command, verbose => $verbose);
     1722            unless ($success) {
     1723                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1724                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1725            }
     1726        }
    15691727    }
    15701728
    … …  
    15741732    die "--stage_id required for stage $stage\n" if !$stage_id;
    15751733    # this stage uses 'camtool'
    1576     my $camtool = can_run('dettool') or die "Can't find dettool";
     1734    my $dettool = can_run('dettool') or die "Can't find dettool";
    15771735
    15781736    # Get list of component imfiles
    … …  
    15851743        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    15861744        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1745    }
     1746    # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this
     1747    if (@$stdout_buf == 0) {
     1748        exit 0;
    15871749    }
    15881750    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    … …  
    16311793       
    16321794        if ($status)  {
    1633             my $command = "$dettool -updateresiddexp -det_id $stage_id -exp_id $exp_id -iteration $iteration";
     1795            my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration";
    16341796            if ($mode eq "goto_cleaned") {
    16351797                $command .= " -data_state cleaned";
    … …  
    16451807                run(command => $command, verbose => $verbose);
    16461808            unless ($success) {
     1809                print STDERR " residexp had an issue setting the state:? $success $error_code\n";
    16471810                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    16481811                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    … …  
    16701833        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    16711834    }
    1672     $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1673         &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1674     $exps = parse_md_list($metadata) or
    1675         &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1676 
    1677     foreach my $exp (@$exps) {
    1678         my $iteration = $exp->{iteration};
    1679         my $command;
    1680         if ($mode eq "goto_cleaned") {
    1681             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1682         }
    1683         if ($mode eq "goto_scrubbed") {
    1684             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1685         }
    1686         if ($mode eq "goto_purged") {
    1687             $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1688         }
    1689         $command .= " -dbname $dbname" if defined $dbname;
    1690         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1691             run(command => $command, verbose => $verbose);
    1692         unless ($success) {
    1693             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1694             &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1695         }
    1696     }
    1697 
     1835    if (@$stdout_buf != 0) {
     1836        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1837            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1838        $exps = parse_md_list($metadata) or
     1839            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1840       
     1841        foreach my $exp (@$exps) {
     1842            my $iteration = $exp->{iteration};
     1843            my $command;
     1844            if ($mode eq "goto_cleaned") {
     1845                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1846            }
     1847            if ($mode eq "goto_scrubbed") {
     1848                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1849            }
     1850            if ($mode eq "goto_purged") {
     1851                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1852            }
     1853            $command .= " -dbname $dbname" if defined $dbname;
     1854            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1855                run(command => $command, verbose => $verbose);
     1856            unless ($success) {
     1857                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1858                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1859            }
     1860        }
     1861    }
    16981862    exit 0;
    16991863}
    … …  
    17061870    my $files = shift; # reference to a list of files to unlink
    17071871
     1872    open(TMPLOG,">>/tmp/czw.cleanup.log");
     1873    flock(TMPLOG,2);
     1874
    17081875    # this script is, of course, very dangerous.
    17091876    foreach my $file (@$files) {
    17101877        print STDERR "unlinking $file\n";
     1878        print TMPLOG "$file\n";
    17111879        $ipprc->file_delete($file);
    17121880    }
     1881
     1882    flock(TMPLOG,8);
     1883    close(TMPLOG);
     1884
    17131885    return 1;
    17141886}
Note: See TracChangeset for help on using the changeset viewer.