- Timestamp:
- May 3, 2010, 8:50:52 AM (16 years ago)
- Location:
- branches/simtest_nebulous_branches
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
ippScripts/scripts/ipp_cleanup.pl (modified) (25 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/simtest_nebulous_branches
- Property svn:mergeinfo changed
-
branches/simtest_nebulous_branches/ippScripts/scripts/ipp_cleanup.pl
r24642 r27840 14 14 use File::Spec; 15 15 use PS::IPP::Config 1.01 qw( :standard ); 16 use PS::IPP::Metadata::List qw( parse_md_list );17 16 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); 18 17 use Pod::Usage qw( pod2usage ); … … 60 59 61 60 62 my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff => 1, 63 detrend.process.imfile => 1, detrend.process.exp => 1, detrend.stack.imfile => 1, 64 detrend.normstat.imfile => 1, detrend.norm.imfile => 1, detrend.norm.exp => 1, 65 detrend.resid.imfile => 1, detrend.resid.exp => 1 ); 66 unless ($stages{$stage}) { 61 my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, 62 "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0, 63 "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 ); 64 unless (exists($stages{$stage})) { 67 65 die "unknown stage $stage for ipp_cleanup.pl\n"; 68 66 } 67 unless (($stages{$stage})) { 68 die "unimplemented stage $stage for ipp_cleanup.pl\n"; 69 } 70 69 71 70 72 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files … … 91 93 92 94 # if there are no chipProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' 95 # XXX Why? This could just mean there's nothing to cleanup, or that we're trying to rerun an errored run. 93 96 if (@$stdout_buf == 0) { 94 my $command = "$chiptool -chip_id $stage_id -updaterun -set_state new"; 95 $command .= " -dbname $dbname" if defined $dbname; 96 97 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 98 run(command => $command, verbose => $verbose); 99 unless ($success) { 100 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 101 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 102 } 103 exit 0; 104 } 105 106 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 97 my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state"; 98 $command .= " -dbname $dbname" if defined $dbname; 99 100 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 101 run(command => $command, verbose => $verbose); 102 unless ($success) { 103 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 104 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 105 } 106 exit 0; 107 } 108 109 # extract the metadata for the files into a hash list 110 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 107 111 &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); 108 109 # extract the metadata for the files into a hash list110 $imfiles = parse_md_list($metadata) or111 &my_die("Unable to parse metadata list", "chip", $stage_id, $PS_EXIT_PROG_ERROR);112 112 113 113 # loop over all of the imfiles, determine the path_base and class_id for each … … 116 116 my $path_base = $imfile->{path_base}; 117 117 my $status = 1; 118 $status = 0 unless defined $path_base and $path_base ne "NULL"; 118 119 119 120 # don't clean up unless the data needed to update is available 120 121 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 121 if ($mode eq "goto_cleaned") { 122 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 123 print STDERR "CHIP: CONFIG_FILE : $config_file\n"; 124 if (!$config_file or ! -e $config_file) { 125 print STDERR "skipping cleanup for chipRun $stage_id $class_id " 126 . " because config file is missing\n"; 127 $status = 0; 122 # goto_scrubbed now requires the config file to not exist. 123 if ($status) { 124 if ($mode eq "goto_cleaned") { 125 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 126 127 unless ($ipprc->file_exists($config_file)) { 128 print STDERR "skipping cleanup for chipRun $stage_id $class_id " 129 . " because config file ($config_file) is missing\n"; 130 $status = 0; 131 } 132 } 133 elsif ($mode eq "goto_scrubbed") { 134 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 135 136 if ($ipprc->file_exists($config_file)) { 137 print STDERR "skipping scrubbed for chipRun $stage_id $class_id " 138 . " because config file ($config_file) is present\n"; 139 $status = 0; 140 } 128 141 } 129 142 } … … 134 147 135 148 # delete the temporary image datafiles 136 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);137 addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);138 addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);149 # addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); 150 # addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); 151 # addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); 139 152 addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id); 140 153 addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id); … … 144 157 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); 145 158 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); 146 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 147 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 159 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); # clean? 160 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); # clean? 148 161 addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); 149 162 addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id); 150 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 163 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); #clean? 151 164 addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); 152 165 } … … 160 173 if ($mode eq "goto_purged") { 161 174 $command .= " -topurgedimfile"; 162 } else { 175 } 176 elsif ($mode eq "goto_cleaned") { 163 177 $command .= " -tocleanedimfile"; 164 178 } 179 elsif ($mode eq "goto_scrubbed") { 180 $command .= " -toscrubbedimfile"; 181 } 182 165 183 $command .= " -dbname $dbname" if defined $dbname; 166 184 … … 171 189 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 172 190 } 191 192 # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. 193 my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; 194 $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned"; 195 $command .= " -dbname $dbname" if defined $dbname; 196 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 197 run(command => $command, verbose => $verbose); 198 unless ($success) { 199 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 200 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); 201 } 202 173 203 } else { 174 204 175 176 177 205 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) 206 my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -set_state $error_state"; 207 $command .= " -dbname $dbname" if defined $dbname; 178 208 179 209 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 183 213 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 184 214 } 215 216 # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. 217 $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state"; 218 $command .= " -dbname $dbname" if defined $dbname; 219 220 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 221 run(command => $command, verbose => $verbose); 222 unless ($success) { 223 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 224 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 225 } 226 185 227 } 186 228 } … … 203 245 &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); 204 246 } 205 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or247 $exps = $mdcParser->parse_list(join "", @$stdout_buf) or 206 248 &my_die("Unable to parse metadata config doc", "camera", $stage_id, $PS_EXIT_PROG_ERROR); 207 208 $exps = parse_md_list($metadata) or209 &my_die("Unable to parse metadata list", "camera", $stage_id, $PS_EXIT_PROG_ERROR);210 249 211 250 my $n_exps = @$exps; … … 218 257 my $status = 1; 219 258 # don't clean up unless the data needed to update is available 259 # goto_scrubbed now requires the config file to not be present 220 260 if ($mode eq "goto_cleaned") { 221 261 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); 222 262 223 if (!$config_file or ! -e $config_file) {263 unless ($ipprc->file_exists($config_file)) { 224 264 print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n"; 265 $status = 0; 266 } 267 } 268 elsif ($mode eq "goto_scrubbed") { 269 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); 270 271 if ($ipprc->file_exists($config_file)) { 272 print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; 225 273 $status = 0; 226 274 } … … 229 277 my @files = (); 230 278 # delete the temporary image datafiles 231 addFilename (\@files, "PSASTRO.OUTPUT", $path_base);279 # addFilename (\@files, "PSASTRO.OUTPUT", $path_base); 232 280 if ($mode eq "goto_purged") { 233 281 # additional files to remove for 'purge' mode … … 244 292 if ($mode eq "goto_cleaned") { 245 293 $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned"; 246 294 } 247 295 if ($mode eq "goto_scrubbed") { 248 $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";249 296 $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed"; 297 } 250 298 if ($mode eq "goto_purged") { 251 299 $command = "$camtool -updaterun -cam_id $stage_id -set_state purged"; … … 258 306 &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); 259 307 } 308 309 # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. 310 my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; 311 $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned"; 312 $command .= " -dbname $dbname" if defined $dbname; 313 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 314 run(command => $command, verbose => $verbose); 315 unless ($success) { 316 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 317 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); 318 } 319 260 320 } else { 261 321 # since 'camera' has only a single imfile, we can just update the run 262 322 my $command = "$camtool -updaterun -cam_id $stage_id -set_state $error_state"; 263 323 $command .= " -dbname $dbname" if defined $dbname; … … 271 331 exit $PS_EXIT_UNKNOWN_ERROR; 272 332 } 333 334 273 335 exit 0; 274 336 } … … 290 352 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); 291 353 } 292 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 354 355 if (@$stdout_buf == 0) { 356 # No skycells were found for some reason. 357 # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." 358 my $command = "$warptool -updaterun -warp_id $stage_id -set_state $error_state"; 359 $command .= " -dbname $dbname" if defined $dbname; 360 361 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 362 run(command => $command, verbose => $verbose); 363 unless ($success) { 364 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 365 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); 366 } 367 368 exit(0); 369 } 370 $skyfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 293 371 &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR); 294 295 $skyfiles = parse_md_list($metadata) or296 &my_die("Unable to parse metadata list", "warp", $stage_id, $PS_EXIT_PROG_ERROR);297 372 298 373 my @files = (); … … 302 377 303 378 my $status = 1; 304 if ($mode eq "goto_cleaned") { 305 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 306 307 if (!$config_file or ! -e $config_file) { 308 print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . 309 " because config file is missing\n"; 310 $status = 0; 311 } 312 } 379 $status = 0 unless defined $path_base and $path_base ne "NULL"; 380 313 381 if ($status) { 314 # delete the temporary image datafiles 315 addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); 316 addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); 317 addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); 318 addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); 319 320 if ($mode eq "goto_purged") { 321 # additional files to remove for 'purge' mode 322 addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); 323 addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); 324 addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); 325 # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); 326 327 # XXX: do we want to delete these? 328 # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); 329 # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); 330 } 382 if ($mode eq "goto_cleaned") { 383 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 384 385 unless ($ipprc->file_exists($config_file)) { 386 print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . 387 " because config file is missing\n"; 388 $status = 0; 389 } 390 } 391 elsif ($mode eq "goto_scrubbed") { 392 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 393 394 if ($ipprc->file_exists($config_file)) { 395 print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . 396 " because config file is present\n"; 397 $status = 0; 398 } 399 } 400 } 401 402 if ($status) { 403 if ($skyfile->{quality} != 8007) { 404 my @files = (); 405 406 # delete the temporary image datafiles 407 addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); 408 addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); 409 addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); 410 # addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); 411 if ($mode eq "goto_purged") { 412 # additional files to remove for 'purge' mode 413 addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); 414 addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); 415 addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); 416 # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); 417 418 # XXX: do we want to delete these? 419 # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); 420 # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); 421 } 331 422 # actual command to delete the files 332 $status = &delete_files (\@files); 423 $status = &delete_files (\@files); 424 } 333 425 } 334 426 … … 337 429 if ($mode eq "goto_purged") { 338 430 $command .= " -topurgedskyfile"; 339 } else { 431 } 432 elsif ($mode eq "goto_cleaned") { 340 433 $command .= " -tocleanedskyfile"; 341 434 } 435 elsif ($mode eq "goto_scrubbed") { 436 $command .= " -toscrubbedskyfile"; 437 } 342 438 $command .= " -dbname $dbname" if defined $dbname; 343 439 … … 348 444 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); 349 445 } 446 447 # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. 448 my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; 449 $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned"; 450 $command .= " -dbname $dbname" if defined $dbname; 451 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 452 run(command => $command, verbose => $verbose); 453 unless ($success) { 454 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 455 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); 456 } 457 350 458 } else { 351 352 459 my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; 460 $command .= " -dbname $dbname" if defined $dbname; 353 461 354 462 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 358 466 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); 359 467 } 360 exit $PS_EXIT_UNKNOWN_ERROR; 468 469 # exit $PS_EXIT_UNKNOWN_ERROR; 361 470 } 362 471 } … … 366 475 ### added for cleanup, based on warp stage entry 367 476 if ($stage eq 'stack') { 368 369 477 die "--stage_id required for stage stack\n" if !$stage_id; 370 371 478 # this stage uses 'stacktool' 372 479 my $stacktool = can_run('stacktool') or die "Can't find stacktool"; … … 378 485 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 379 486 unless ($success) { 380 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 381 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 382 } 383 my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 384 &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR); 385 386 $skyfiles = parse_md_list($metadata) or 387 &my_die("Unable to parse metadata list", "stack", $stage_id, $PS_EXIT_PROG_ERROR); 388 487 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 488 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 489 } 490 491 if (@$stdout_buf == 0) { 492 # No skycells were found for some reason. 493 # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." 494 my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state"; 495 $command .= " -dbname $dbname" if defined $dbname; 496 497 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 498 run(command => $command, verbose => $verbose); 499 unless ($success) { 500 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 501 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 502 } 503 504 exit(0); 505 } 506 507 $skyfiles = $mdcParser->parse_list(join "", @{ $stdout_buf }) or 508 &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR); 509 389 510 my @files = (); 390 511 foreach my $skyfile (@{ $skyfiles }) { 391 my $path_base = $skyfile->{path_base}; 392 my $skycell_id = $skyfile->{skycell_id}; 393 394 my $status = 1; 395 if ($mode eq "goto_cleaned") { 396 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); 397 print STDERR "MY CONFIG FILE = $config_file\n"; 398 printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); 399 $config_file =~ s%^file://%%; 400 if (!$config_file or ! -e $config_file) { 401 print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" . 402 " because config file is missing\n"; 403 $status = 0; 404 } 405 $config_file = 'file://' . $config_file; 406 } 407 if ($status) { 408 # delete the temporary image datafiles 409 addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id); 410 addFilename(\@files, "PPSTACK.OUTPUT.MASK", $path_base, $skycell_id); 411 addFilename(\@files, "PPSTACK.OUTPUT.VARIANCE", $path_base, $skycell_id); 412 413 if ($mode eq "goto_purged") { 414 # additional files to remove for 'purge' mode 415 addFilename(\@files, "PPSTACK.CONV.KERNEL", $path_base, $skycell_id); 416 addFilename(\@files, "PPSTACK.OUTPUT.JPEG1", $path_base, $skycell_id); 417 addFilename(\@files, "PPSTACK.OUTPUT.JPEG2", $path_base, $skycell_id); 418 # Commented out to match warp files. 419 #addFilename(\@files, "PPSTACK.TARGET.PSF", $path_base, $skycell_id); 420 #addFilename(\@files, "PPSTACK.CONFIG", $path_base, $skycell_id); 421 } 422 423 $status = &delete_files(\@files); 424 } 425 426 if ($status) { 427 my $command = "$stacktool -stack_id $stage_id"; 428 if ($mode eq "goto_purged") { 429 $command .= " -updaterun -state purged"; 430 } else { 431 $command .= " -updaterun -state cleaned"; 432 } 433 $command .= " -dbname $dbname" if defined $dbname; 434 435 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 436 run(command => $command, verbose => $verbose); 437 unless ($success) { 438 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 439 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 440 } 441 } else { 442 my $command = "$stacktool -updaterun -stack_id $stage_id -state $error_state"; 443 $command .= " -dbname $dbname" if defined $dbname; 444 445 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 446 run(command => $command, verbose => $verbose); 447 unless ($success) { 448 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 449 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 450 } 451 exit $PS_EXIT_UNKNOWN_ERROR; 452 } 512 my $path_base = $skyfile->{path_base}; 513 my $skycell_id = $skyfile->{skycell_id}; 514 515 my $status = 1; 516 if ((!exists($skyfile->{path_base}))|| 517 (!defined($path_base))) { 518 $status = 0; 519 } 520 if ($status) { 521 if ($mode eq "goto_cleaned") { 522 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); 523 524 unless ($ipprc->file_exists($config_file)) { 525 print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" . 526 " because config file is missing\n"; 527 $status = 0; 528 } 529 } 530 elsif ($mode eq "goto_scrubbed") { 531 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); 532 533 if ($ipprc->file_exists($config_file)) { 534 print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . 535 " because config file is present\n"; 536 $status = 0; 537 } 538 } 539 } 540 541 if ($status) { 542 my @files = (); 543 # delete the temporary image datafiles 544 addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id); 545 addFilename(\@files, "PPSTACK.OUTPUT.MASK", $path_base, $skycell_id); 546 addFilename(\@files, "PPSTACK.OUTPUT.VARIANCE", $path_base, $skycell_id); 547 548 if ($mode eq "goto_purged") { 549 # additional files to remove for 'purge' mode 550 addFilename(\@files, "PPSTACK.CONV.KERNEL", $path_base, $skycell_id); 551 addFilename(\@files, "PPSTACK.OUTPUT.JPEG1", $path_base, $skycell_id); 552 addFilename(\@files, "PPSTACK.OUTPUT.JPEG2", $path_base, $skycell_id); 553 # Commented out to match warp files. 554 #addFilename(\@files, "PPSTACK.TARGET.PSF", $path_base, $skycell_id); 555 #addFilename(\@files, "PPSTACK.CONFIG", $path_base, $skycell_id); 556 } 557 558 $status = &delete_files(\@files); 559 } 560 561 if ($status) { 562 my $command = "$stacktool -stack_id $stage_id"; 563 if ($mode eq "goto_purged") { 564 $command .= " -updaterun -set_state purged"; 565 } 566 elsif ($mode eq "goto_cleaned") { 567 $command .= " -updaterun -set_state cleaned"; 568 } 569 elsif ($mode eq "goto_scrubbed") { 570 $command .= " -updaterun -set_state scrubbed"; 571 } 572 $command .= " -dbname $dbname" if defined $dbname; 573 574 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 575 run(command => $command, verbose => $verbose); 576 unless ($success) { 577 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 578 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 579 } 580 581 # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. 582 my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; 583 $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned"; 584 $command .= " -dbname $dbname" if defined $dbname; 585 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 586 run(command => $command, verbose => $verbose); 587 unless ($success) { 588 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 589 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); 590 } 591 592 } else { 593 my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state"; 594 $command .= " -dbname $dbname" if defined $dbname; 595 596 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 597 run(command => $command, verbose => $verbose); 598 unless ($success) { 599 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 600 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 601 } 602 # exit $PS_EXIT_UNKNOWN_ERROR; 603 } 453 604 } 454 605 exit 0; … … 456 607 457 608 if ($stage eq 'diff') { 458 609 459 610 die "--stage_id required for stage diff\n" if !$stage_id; 460 611 … … 468 619 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 469 620 unless ($success) { 470 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 471 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 472 } 473 my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 474 &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); 475 476 $skyfiles = parse_md_list($metadata) or 477 &my_die("Unable to parse metadata list", "diff", $stage_id, $PS_EXIT_PROG_ERROR); 478 621 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 622 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 623 } 624 625 if (@$stdout_buf == 0) { 626 # No skycells were found for some reason. 627 # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." 628 my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; 629 $command .= " -dbname $dbname" if defined $dbname; 630 631 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 632 run(command => $command, verbose => $verbose); 633 unless ($success) { 634 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 635 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 636 } 637 638 exit(0); 639 } 640 641 $skyfiles = $mdcParser->parse_list(join "", @{ $stdout_buf }) or 642 &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); 643 479 644 my @files = (); 480 645 foreach my $skyfile (@{ $skyfiles }) { 481 my $path_base = $skyfile->{path_base}; 482 my $skycell_id = $skyfile->{skycell_id}; 483 484 my $status = 1; 485 if ($mode eq "goto_cleaned") { 486 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); 487 print STDERR "MY CONFIG FILE = $config_file\n"; 488 printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); 489 $config_file =~ s%^file://%%; 490 if (!$config_file or ! -e $config_file) { 491 print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" . 492 " because config file ($config_file) is missing\n"; 493 $status = 0; 494 } 495 $config_file = 'file://' . $config_file; 496 } 497 if ($status) { 498 # delete the temporary image datafiles 499 addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id); 500 addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id); 501 addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id); 502 503 addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id); 504 addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id); 505 addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id); 506 addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id); 507 addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id); 508 addFilename(\@files, "PPSUB.INPUT.CONV.VARIANCE", $path_base, $skycell_id); 509 510 addFilename(\@files, "PPSUB.REF.CONV", $path_base, $skycell_id); 511 addFilename(\@files, "PPSUB.REF.CONV.MASK", $path_base, $skycell_id); 512 addFilename(\@files, "PPSUB.REF.CONV.VARIANCE", $path_base, $skycell_id); 513 514 if ($mode eq "goto_purged") { 515 # additional files to remove for 'purge' mode 516 addFilename(\@files, "PPSUB.OUTPUT.KERNELS", $path_base, $skycell_id); 517 addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id); 518 addFilename(\@files, "PPSUB.OUTPUT.JPEG2", $path_base, $skycell_id); 519 # Commented out to match warp files. 520 #addFilename(\@files, "PPSUB.CONFIG", $path_base, $skycell_id); 521 addFilename(\@files, "PPSUB.OUTPUT.SOURCES", $path_base, $skycell_id); 522 addFilename(\@files, "PPSUB.INVERSE.SOURCES", $path_base, $skycell_id); 523 524 } 525 print STDERR "MY FILES: @files\n"; 526 $status = &delete_files(\@files); 527 } 528 print STDERR "MY STATUS: $status\n"; 529 if ($status) { 530 my $command = "$difftool -diff_id $stage_id"; 531 # my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; 532 if ($mode eq "goto_purged") { 533 $command .= " -updaterun -state purged"; 534 } else { 535 $command .= " -updaterun -state cleaned"; 536 } 537 $command .= " -dbname $dbname" if defined $dbname; 538 539 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 540 run(command => $command, verbose => $verbose); 541 unless ($success) { 542 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 543 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 544 } 545 } else { 546 my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state"; 547 # my $command = "$difftool -updaterun -diff_id $stage_id -skycell_id $skycell_id -state $error_state"; 548 $command .= " -dbname $dbname" if defined $dbname; 549 550 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 551 run(command => $command, verbose => $verbose); 552 unless ($success) { 553 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 554 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 555 } 556 exit $PS_EXIT_UNKNOWN_ERROR; 557 } 646 my $path_base = $skyfile->{path_base}; 647 my $skycell_id = $skyfile->{skycell_id}; 648 649 my $status = 1; 650 if ((!exists($skyfile->{path_base}))|| 651 (!defined($path_base))) { 652 $status = 0; 653 } 654 if ($status) { 655 656 if ($mode eq "goto_cleaned") { 657 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); 658 659 unless ($ipprc->file_exists($config_file)) { 660 print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" . 661 " because config file ($config_file) is missing\n"; 662 $status = 0; 663 } 664 } 665 elsif ($mode eq "goto_scrubbed") { 666 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); 667 668 if ($ipprc->file_exists($config_file)) { 669 print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . 670 " because config file ($config_file) is present\n"; 671 $status = 0; 672 } 673 } 674 } 675 if ($status) { 676 my @files = (); 677 # delete the temporary image datafiles 678 addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id); 679 addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id); 680 addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id); 681 682 addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id); 683 addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id); 684 addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id); 685 686 addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id); 687 addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id); 688 addFilename(\@files, "PPSUB.INPUT.CONV.VARIANCE", $path_base, $skycell_id); 689 690 addFilename(\@files, "PPSUB.REF.CONV", $path_base, $skycell_id); 691 addFilename(\@files, "PPSUB.REF.CONV.MASK", $path_base, $skycell_id); 692 addFilename(\@files, "PPSUB.REF.CONV.VARIANCE", $path_base, $skycell_id); 693 694 if ($mode eq "goto_purged") { 695 # additional files to remove for 'purge' mode 696 addFilename(\@files, "PPSUB.OUTPUT.KERNELS", $path_base, $skycell_id); 697 addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id); 698 addFilename(\@files, "PPSUB.OUTPUT.JPEG2", $path_base, $skycell_id); 699 # Commented out to match warp files. 700 #addFilename(\@files, "PPSUB.CONFIG", $path_base, $skycell_id); 701 addFilename(\@files, "PPSUB.OUTPUT.SOURCES", $path_base, $skycell_id); 702 addFilename(\@files, "PPSUB.INVERSE.SOURCES", $path_base, $skycell_id); 703 704 } 705 # print STDERR "MY FILES: @files\n"; 706 $status = &delete_files(\@files); 707 } 708 # print STDERR "MY STATUS: $status\n"; 709 if ($status) { 710 my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; 711 712 if ($mode eq "goto_purged") { 713 $command .= " -topurgedskyfile"; 714 } 715 elsif ($mode eq "goto_cleaned") { 716 $command .= " -tocleanedskyfile"; 717 } 718 elsif ($mode eq "goto_scrubbed") { 719 $command .= " -toscrubbedskyfile"; 720 } 721 722 $command .= " -dbname $dbname" if defined $dbname; 723 724 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 725 run(command => $command, verbose => $verbose); 726 unless ($success) { 727 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 728 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 729 } 730 731 # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. 732 my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; 733 $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned"; 734 $command .= " -dbname $dbname" if defined $dbname; 735 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 736 run(command => $command, verbose => $verbose); 737 unless ($success) { 738 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 739 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); 740 } 741 742 } else { 743 my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; 744 745 $command .= " -dbname $dbname" if defined $dbname; 746 747 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 748 run(command => $command, verbose => $verbose); 749 unless ($success) { 750 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 751 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 752 } 753 # exit $PS_EXIT_UNKNOWN_ERROR; 754 } 558 755 } 559 756 exit 0; 560 757 } 561 758 if ($stage eq 'fake') { 562 die "ipp_cleanup.pl -stage fake not yet implemented. Probably will just mark database cleaned.\n"; 563 } 564 # fake : faketool : -pendingcleanupimfile (loop over imfiles) 759 print STDERR "This does not seem to work at present, as no files exist. Terminating quietly.\n"; 760 exit(0); 761 die "--stage_id required for stage fake\n" if !$stage_id; 762 ### select the imfiles for this entry 763 764 # this stage uses 'chiptool' 765 my $faketool = can_run('faketool') or die "Can't find faketool"; 766 767 # Get list of component imfiles 768 # XXX may need a different my_die for each stage 769 my $imfiles; # Array of component files 770 my $command = "$faketool -pendingcleanupimfile -fake_id $stage_id"; # Command to run 771 $command .= " -dbname $dbname" if defined $dbname; 772 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 773 unless ($success) { 774 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 775 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 776 } 777 778 # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' 779 if (@$stdout_buf == 0) { 780 my $command = "$faketool -fake_id $stage_id -updaterun -set_state $error_state"; 781 $command .= " -dbname $dbname" if defined $dbname; 782 783 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 784 run(command => $command, verbose => $verbose); 785 unless ($success) { 786 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 787 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 788 } 789 exit 0; 790 } 791 792 # extract the metadata for the files into a hash list 793 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 794 &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR); 795 796 # loop over all of the imfiles, determine the path_base and class_id for each 797 foreach my $imfile (@$imfiles) { 798 my $class_id = $imfile->{class_id}; 799 my $path_base = $imfile->{path_base}; 800 my $status = 1; 801 802 # don't clean up unless the data needed to update is available 803 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 804 # goto_scrubbed now requires the config file to not exist. 805 if ($mode eq "goto_cleaned") { 806 my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); 807 808 unless ($ipprc->file_exists($config_file)) { 809 print STDERR "skipping cleanup for fakeRun $stage_id $class_id " 810 . " because config file is missing\n"; 811 $status = 0; 812 } 813 } 814 elsif ($mode eq "goto_scrubbed") { 815 my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); 816 817 if ($ipprc->file_exists($config_file)) { 818 print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " 819 . " because config file is present\n"; 820 $status = 0; 821 } 822 } 823 824 if ($status) { 825 # array of actual filenames to delete 826 my @files = (); 827 828 # delete the temporary image datafiles 829 addFilename (\@files, "PPSIM.OUTPUT.MEF", $path_base, $class_id); 830 addFilename (\@files, "PPSIM.OUTPUT.SPL", $path_base, $class_id); 831 addFilename (\@files, "PPSIM.FAKE.CHIP", $path_base, $class_id); 832 addFilename (\@files, "PPSIM.FORCE.CHIP", $path_base, $class_id); 833 if ($mode eq "goto_purged") { 834 # additional files to remove for 'purge' mode 835 addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id); 836 addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id); 837 addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id); 838 } 839 840 # actual command to delete the files 841 $status = &delete_files (\@files); 842 } 843 844 if ($status) { 845 my $command = "$faketool -fake_id $stage_id -class_id $class_id"; 846 if ($mode eq "goto_purged") { 847 $command .= " -topurgedimfile"; 848 } 849 elsif ($mode eq "goto_cleaned") { 850 $command .= " -tocleanedimfile"; 851 } 852 elsif ($mode eq "goto_scrubbed") { 853 $command .= " -toscrubbedimfile"; 854 } 855 856 $command .= " -dbname $dbname" if defined $dbname; 857 858 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 859 run(command => $command, verbose => $verbose); 860 unless ($success) { 861 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 862 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 863 } 864 } else { 865 866 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) 867 my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state"; 868 $command .= " -dbname $dbname" if defined $dbname; 869 870 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 871 run(command => $command, verbose => $verbose); 872 unless ($success) { 873 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 874 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 875 } 876 } 877 } 878 exit 0; 879 880 } 881 # Detrend stages 882 if ($stage eq "detrend.processed") { 883 884 die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id; 885 ### select the imfiles for this entry 886 887 # Neither det_id nor exp_id uniquely determine a det exposure, so we pack them. 888 my ($det_id,$exp_id) = split /\./, $stage_id; #/ trailing slash for emacs; 889 890 # this stage uses 'dettool' 891 my $dettool = can_run('dettool') or die "Can't find chiptool"; 892 893 # Get list of component imfiles 894 # XXX may need a different my_die for each stage 895 my $imfiles; # Array of component files 896 my $metadata; 897 my $command = "$dettool -pendingcleanup_processedimfile -det_id $det_id -exp_id $exp_id"; # Command to run 898 $command .= " -dbname $dbname" if defined $dbname; 899 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 900 unless ($success) { 901 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 902 &my_die("Unable to perform dettool: $error_code", "detrend.processed", $stage_id, $error_code); 903 } 904 905 # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' 906 if (@$stdout_buf != 0) { 907 # exit 0; # Silently exit if there's nothing to do. I don't know how we'd ever get here, but let's be safe. 908 909 910 # extract the metadata for the files into a hash list 911 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 912 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 913 } 914 # loop over all of the imfiles, determine the path_base and class_id for each 915 foreach my $imfile (@$imfiles) { 916 my $iexp_id = $imfile->{exp_id}; 917 my $class_id = $imfile->{class_id}; 918 my $path_base = $imfile->{path_base}; 919 my $status = 1; 920 921 unless (defined($path_base)) { 922 print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n"; 923 $status = 0; 924 } 925 unless (defined($class_id)) { 926 print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n"; 927 $status = 0; 928 } 929 # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent, 930 # and so there is no check for config files. 931 if ($status) { 932 # array of actual filenames to delete 933 my @files = (); 934 # delete the temporary image datafiles 935 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); 936 addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); 937 addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); 938 if ($mode eq "goto_purged") { 939 # additional files to remove for 'purge' mode 940 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 941 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 942 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 943 } 944 # actual command to delete the files 945 $status = &delete_files (\@files); 946 } 947 948 if ($status) { 949 my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile"; 950 if ($mode eq "goto_purged") { 951 $command .= " -data_state purged"; 952 } 953 elsif ($mode eq "goto_cleaned") { 954 $command .= " -data_state cleaned"; 955 } 956 elsif ($mode eq "goto_scrubbed") { 957 $command .= " -data_state scrubbed"; 958 } 959 $command .= " -dbname $dbname" if defined $dbname; 960 961 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 962 run(command => $command, verbose => $verbose); 963 unless ($success) { 964 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 965 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 966 } 967 968 } else { 969 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* 970 my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile "; 971 $command .= " -data_state $error_state"; 972 $command .= " -dbname $dbname" if defined $dbname; 973 974 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 975 run(command => $command, verbose => $verbose); 976 unless ($success) { 977 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 978 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 979 } 980 } 981 } 982 983 # Flag the detProcessedExp as clean now as well, if it is marked to be cleaned (this is clunky, but works for now). 984 985 $command = "$dettool -pendingcleanup_processedexp -det_id $det_id -exp_id $exp_id"; 986 $command .= " -dbname $dbname" if defined $dbname; 987 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 988 unless ($success) { 989 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 990 &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code); 991 } 992 if (@$stdout_buf != 0) { 993 my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or 994 &my_die("Unable to parse metadata config doc", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); 995 996 foreach my $exp (@$exps) { 997 my $exp_id = $exp->{exp_id}; 998 my $command = "$dettool -updateprocessedexp -det_id $det_id -exp_id $exp_id "; 999 if ($mode eq "goto_cleaned") { 1000 $command .= " -data_state cleaned "; 1001 } 1002 if ($mode eq "goto_scrubbed") { 1003 $command .= " -data_state scrubbed "; 1004 } 1005 if ($mode eq "goto_purged") { 1006 $command .= " -data_state purged "; 1007 } 1008 $command .= " -dbname $dbname" if defined $dbname; 1009 # print "$command\n"; 1010 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1011 run(command => $command, verbose => $verbose); 1012 unless ($success) { 1013 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1014 &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code); 1015 } 1016 } 1017 } 1018 1019 exit 0; 1020 } 1021 if ($stage eq "detrend.resid") { 1022 1023 die "--stage_id required for stage $stage\n" if !$stage_id; 1024 ### select the imfiles for this entry 1025 1026 # Neither det_id nor exp_id uniquely determine the det exposure, so we pack them 1027 my ($det_id,$exp_id) = split /\./, $stage_id; #/ trailing slash for emacs; 1028 1029 # this stage uses 'dettool' 1030 my $dettool = can_run('dettool') or die "Can't find dettool"; 1031 1032 # Get list of component imfiles 1033 # XXX may need a different my_die for each stage 1034 my $imfiles; # Array of component files 1035 my $metadata; 1036 my $command = "$dettool -pendingcleanup_residimfile -det_id $det_id -exp_id $exp_id"; # Command to run 1037 $command .= " -dbname $dbname" if defined $dbname; 1038 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1039 unless ($success) { 1040 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1041 &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); 1042 } 1043 1044 # if there are no detResidImfiles (@$stdout_buf == 0), then silently exit. 1045 if (@$stdout_buf != 0) { 1046 # extract the metadata for the files into a hash list 1047 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 1048 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1049 } 1050 # loop over all of the imfiles, determine the path_base and class_id for each 1051 foreach my $imfile (@$imfiles) { 1052 my $iexp_id = $imfile->{exp_id}; 1053 my $class_id = $imfile->{class_id}; 1054 my $path_base = $imfile->{path_base}; 1055 my $iteration = $imfile->{iteration}; 1056 1057 my $status = 1; 1058 1059 # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent, 1060 # and so there is no check for config files. 1061 unless (defined($path_base)) { 1062 print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n"; 1063 $status = 0; 1064 } 1065 unless (defined($class_id)) { 1066 print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n"; 1067 $status = 0; 1068 } 1069 if ($status) { 1070 # array of actual filenames to delete 1071 my @files = (); 1072 1073 # delete the temporary image datafiles 1074 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); 1075 if ($mode eq "goto_purged") { 1076 # additional files to remove for 'purge' mode 1077 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 1078 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 1079 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 1080 } 1081 # foreach my $f (@files) { 1082 # print "RESID: $f\n"; 1083 # } 1084 # actual command to delete the files 1085 $status = &delete_files (\@files); 1086 } 1087 1088 if ($status) { 1089 my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile "; 1090 if ($mode eq "goto_purged") { 1091 $command .= " -data_state purged"; 1092 } 1093 elsif ($mode eq "goto_cleaned") { 1094 $command .= " -data_state cleaned"; 1095 } 1096 elsif ($mode eq "goto_scrubbed") { 1097 $command .= " -data_state scrubbed"; 1098 } 1099 1100 $command .= " -dbname $dbname" if defined $dbname; 1101 1102 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1103 run(command => $command, verbose => $verbose); 1104 unless ($success) { 1105 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1106 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1107 } 1108 } else { 1109 my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile "; 1110 $command .= " -data_state $error_state "; 1111 $command .= " -dbname $dbname" if defined $dbname; 1112 1113 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1114 run(command => $command, verbose => $verbose); 1115 unless ($success) { 1116 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1117 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1118 } 1119 } 1120 } 1121 # Flag the detResidExp as clean now as well, if it is marked tobe cleaned (this is still clunky). 1122 1123 $command = "$dettool -pendingcleanup_residexp -det_id $det_id -exp_id $exp_id"; 1124 $command .= " -dbname $dbname" if defined $dbname; 1125 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1126 unless ($success) { 1127 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1128 &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code); 1129 } 1130 if (@$stdout_buf != 0) { 1131 my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or 1132 &my_die("Unable to parse metadata config doc", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); 1133 1134 foreach my $exp (@$exps) { 1135 my $iteration = $exp->{iteration}; 1136 my $command = "$dettool -updateresidexp -det_id $det_id -exp_id $exp_id "; 1137 if ($mode eq "goto_cleaned") { 1138 $command .= " -data_state cleaned"; 1139 } 1140 if ($mode eq "goto_scrubbed") { 1141 $command .= " -data_state scrubbed"; 1142 } 1143 if ($mode eq "goto_purged") { 1144 $command .= " -data_state purged"; 1145 } 1146 $command .= " -dbname $dbname" if defined $dbname; 1147 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1148 run(command => $command, verbose => $verbose); 1149 unless ($success) { 1150 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1151 &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code); 1152 } 1153 } 1154 } 1155 1156 exit 0; 1157 } 1158 1159 if ($stage eq "detrend.stack.imfile") { 1160 1161 die "--stage_id required for stage $stage\n" if !$stage_id; 1162 1163 # this stage uses 'dettool' 1164 my $dettool = can_run('dettool') or die "Can't find dettool"; 1165 1166 # Get list of component imfiles 1167 my $stacks; # Array reference of component files 1168 my $command = "$dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run 1169 $command .= " -dbname $dbname" if defined $dbname; 1170 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1171 unless ($success) { 1172 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1173 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1174 } 1175 $stacks = $mdcParser->parse_list(join "", @{ $stdout_buf }) or 1176 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1177 1178 my @files = (); 1179 foreach my $stack (@{ $stacks }) { 1180 # detStackedImfile does not have a path_base column. This is inconvenient, as it means we need to calculate it. 1181 my $path_base = $stack->{uri}; 1182 my $iteration = $stack->{iteration}; 1183 my $class_id = $stack->{class_id}; 1184 1185 $path_base =~ s/\.fits$//; # That should do it? 1186 1187 my $status = 1; 1188 1189 if ($status) { 1190 my @files = (); 1191 # delete the temporary image datafiles 1192 # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. 1193 # I understand that they all point to the same filename right now, but that may not be true in 1194 # the future. 1195 addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id); 1196 addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id); 1197 addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id); 1198 addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id); 1199 addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id); 1200 addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id); 1201 1202 1203 addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id); 1204 addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id); 1205 1206 if ($mode eq "goto_purged") { 1207 # additional files to remove for 'purge' mode 1208 # addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id); 1209 } 1210 1211 $status = &delete_files(\@files); 1212 } 1213 1214 if ($status) { 1215 my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id"; 1216 if ($mode eq "goto_purged") { 1217 $command .= " -updatestacked -data_state purged"; 1218 } 1219 elsif ($mode eq "goto_cleaned") { 1220 $command .= " -updatestacked -data_state cleaned"; 1221 } 1222 elsif ($mode eq "goto_scrubbed") { 1223 $command .= " -updatestacked -data_state scrubbed"; 1224 } 1225 $command .= " -dbname $dbname" if defined $dbname; 1226 1227 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1228 run(command => $command, verbose => $verbose); 1229 unless ($success) { 1230 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1231 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1232 } 1233 } else { 1234 my $command = "$dettool -updatestacked -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1235 $command .= " -dbname $dbname" if defined $dbname; 1236 1237 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1238 run(command => $command, verbose => $verbose); 1239 unless ($success) { 1240 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1241 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1242 } 1243 exit $PS_EXIT_UNKNOWN_ERROR; 1244 } 1245 } 1246 # Check to see if we can mark the whole detRunSummary object as cleaned. 1247 1248 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1249 $command .= " -dbname $dbname" if defined $dbname; 1250 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1251 unless ($success) { 1252 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1253 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1254 } 1255 if (@$stdout_buf != 0) { 1256 my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or 1257 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1258 1259 foreach my $exp (@$exps) { 1260 my $iteration = $exp->{iteration}; 1261 my $command; 1262 if ($mode eq "goto_cleaned") { 1263 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1264 } 1265 if ($mode eq "goto_scrubbed") { 1266 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1267 } 1268 if ($mode eq "goto_purged") { 1269 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1270 } 1271 $command .= " -dbname $dbname" if defined $dbname; 1272 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1273 run(command => $command, verbose => $verbose); 1274 unless ($success) { 1275 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1276 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1277 } 1278 } 1279 } 1280 exit 0; 1281 } 1282 if ($stage eq "detrend.normstat.imfile") { 1283 print STDERR "I'm not convinced there's anything to clean up from stage $stage\n"; 1284 die "--stage_id required for stage $stage\n" if !$stage_id; 1285 # this stage uses 'camtool' 1286 my $dettool = can_run('dettool') or die "Can't find dettool"; 1287 1288 my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run 1289 $command .= " -dbname $dbname" if defined $dbname; 1290 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1291 unless ($success) { 1292 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1293 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1294 } 1295 my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or 1296 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1297 1298 foreach my $exp (@$exps) { 1299 # my $path_base = $exp->{path_base}; 1300 my $iteration = $exp->{iteration}; 1301 my $class_id = $exp->{class_id}; 1302 1303 my $status = 1; 1304 if ($status) { 1305 my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id"; 1306 if ($mode eq "goto_cleaned") { 1307 $command .= " -data_state cleaned"; 1308 } 1309 if ($mode eq "goto_scrubbed") { 1310 $command .= " -data_state scrubbed"; 1311 } 1312 if ($mode eq "goto_purged") { 1313 $command .= " -data_state purged"; 1314 } 1315 $command .= " -dbname $dbname" if defined $dbname; 1316 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1317 run(command => $command, verbose => $verbose); 1318 unless ($success) { 1319 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1320 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1321 } 1322 } else { 1323 my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1324 $command .= " -dbname $dbname" if defined $dbname; 1325 1326 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1327 run(command => $command, verbose => $verbose); 1328 unless ($success) { 1329 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1330 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1331 } 1332 exit $PS_EXIT_UNKNOWN_ERROR; 1333 } 1334 } 1335 # Check to see if we can mark the whole detRunSummary object as cleaned. 1336 1337 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1338 $command .= " -dbname $dbname" if defined $dbname; 1339 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1340 unless ($success) { 1341 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1342 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1343 } 1344 if (@$stdout_buf != 0) { 1345 $exps = $mdcParser->parse_list(join "", @$stdout_buf) or 1346 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1347 1348 foreach my $exp (@$exps) { 1349 my $iteration = $exp->{iteration}; 1350 my $command; 1351 if ($mode eq "goto_cleaned") { 1352 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1353 } 1354 if ($mode eq "goto_scrubbed") { 1355 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1356 } 1357 if ($mode eq "goto_purged") { 1358 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1359 } 1360 $command .= " -dbname $dbname" if defined $dbname; 1361 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1362 run(command => $command, verbose => $verbose); 1363 unless ($success) { 1364 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1365 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1366 } 1367 } 1368 } 1369 1370 exit 0; 1371 1372 } 1373 if ($stage eq "detrend.norm.imfile") { 1374 die "--stage_id required for stage $stage\n" if !$stage_id; 1375 # this stage uses 'dettool' 1376 my $dettool = can_run('dettool') or die "Can't find dettool"; 1377 1378 # Get list of component imfiles 1379 # XXX may need a different my_die for each stage 1380 my $exps; # Array of component files 1381 my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run 1382 $command .= " -dbname $dbname" if defined $dbname; 1383 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1384 unless ($success) { 1385 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1386 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1387 } 1388 $exps = $mdcParser->parse_list(join "", @$stdout_buf) or 1389 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1390 1391 foreach my $exp (@$exps) { 1392 my $path_base = $exp->{path_base}; 1393 my $iteration = $exp->{iteration}; 1394 my $class_id = $exp->{class_id}; 1395 1396 my $status = 1; 1397 # don't clean up unless the data needed to update is available 1398 # goto_scrubbed now requires the config file to not be present 1399 if ($mode eq "goto_cleaned") { 1400 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1401 1402 unless ($ipprc->file_exists($config_file)) { 1403 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 1404 $status = 0; 1405 } 1406 } 1407 elsif ($mode eq "goto_scrubbed") { 1408 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1409 1410 if ($ipprc->file_exists($config_file)) { 1411 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 1412 $status = 0; 1413 } 1414 } 1415 if ($status) { 1416 my @files = (); 1417 1418 if ($mode eq "goto_purged") { 1419 # additional files to remove for 'purge' mode 1420 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base); 1421 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base); 1422 1423 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base); 1424 addFilename (\@files, "PPIMAGE.STATS", $path_base); 1425 } 1426 # actual command to delete the files 1427 $status = &delete_files (\@files); 1428 } 1429 1430 if ($status) { 1431 my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id"; 1432 if ($mode eq "goto_cleaned") { 1433 $command .= " -data_state cleaned"; 1434 } 1435 if ($mode eq "goto_scrubbed") { 1436 $command .= " -data_state scrubbed"; 1437 } 1438 if ($mode eq "goto_purged") { 1439 $command .= " -data_state purged"; 1440 } 1441 $command .= " -dbname $dbname" if defined $dbname; 1442 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1443 run(command => $command, verbose => $verbose); 1444 unless ($success) { 1445 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1446 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1447 } 1448 } else { 1449 my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1450 $command .= " -dbname $dbname" if defined $dbname; 1451 1452 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1453 run(command => $command, verbose => $verbose); 1454 unless ($success) { 1455 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1456 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1457 } 1458 exit $PS_EXIT_UNKNOWN_ERROR; 1459 } 1460 } 1461 # Check to see if we can mark the whole detRunSummary object as cleaned. 1462 1463 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1464 $command .= " -dbname $dbname" if defined $dbname; 1465 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1466 unless ($success) { 1467 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1468 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1469 } 1470 if (@$stdout_buf != 0) { 1471 $exps = $mdcParser->parse_list(join "", @$stdout_buf) or 1472 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1473 1474 foreach my $exp (@$exps) { 1475 my $iteration = $exp->{iteration}; 1476 my $command; 1477 if ($mode eq "goto_cleaned") { 1478 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1479 } 1480 if ($mode eq "goto_scrubbed") { 1481 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1482 } 1483 if ($mode eq "goto_purged") { 1484 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1485 } 1486 $command .= " -dbname $dbname" if defined $dbname; 1487 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1488 run(command => $command, verbose => $verbose); 1489 unless ($success) { 1490 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1491 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1492 } 1493 } 1494 } 1495 1496 exit 0; 1497 } 1498 if ($stage eq "detrend.norm.exp") { 1499 die "--stage_id required for stage $stage\n" if !$stage_id; 1500 # this stage uses 'dettool' 1501 my $dettool = can_run('dettool') or die "Can't find dettool"; 1502 1503 # Get list of component imfiles 1504 # XXX may need a different my_die for each stage 1505 my $exps; # Array of component files 1506 my $command = "$dettool -pendingcleanup_normalizedexp -det_id $stage_id"; # Command to run 1507 $command .= " -dbname $dbname" if defined $dbname; 1508 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1509 unless ($success) { 1510 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1511 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1512 } 1513 1514 if (@$stdout_buf == 0) { 1515 exit 0; 1516 } 1517 $exps = $mdcParser->parse_list(join "", @$stdout_buf) or 1518 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1519 1520 foreach my $exp (@$exps) { 1521 my $exp_id = $exp->{exp_id}; 1522 my $iteration = $exp->{iteration}; 1523 my $path_base = $exp->{path_base}; 1524 1525 my $status = 1; 1526 # don't clean up unless the data needed to update is available 1527 # goto_scrubbed now requires the config file to not be present 1528 if ($mode eq "goto_cleaned") { 1529 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1530 1531 unless ($ipprc->file_exists($config_file)) { 1532 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 1533 $status = 0; 1534 } 1535 } 1536 elsif ($mode eq "goto_scrubbed") { 1537 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1538 1539 if ($ipprc->file_exists($config_file)) { 1540 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 1541 $status = 0; 1542 } 1543 } 1544 if ($status) { 1545 my @files = (); 1546 # delete the temporary image datafiles 1547 if ($mode eq "goto_purged") { 1548 # additional files to remove for 'purge' mode 1549 addFilename (\@files, "PPIMAGE.JPEG1", $path_base); 1550 addFilename (\@files, "PPIMAGE.JPEG2", $path_base); 1551 } 1552 # actual command to delete the files 1553 $status = &delete_files (\@files); 1554 } 1555 1556 if ($status) { 1557 my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration"; 1558 if ($mode eq "goto_cleaned") { 1559 $command .= " -data_state cleaned"; 1560 } 1561 if ($mode eq "goto_scrubbed") { 1562 $command .= " -data_state scrubbed"; 1563 } 1564 if ($mode eq "goto_purged") { 1565 $command .= " -data_state purged"; 1566 } 1567 $command .= " -dbname $dbname" if defined $dbname; 1568 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1569 run(command => $command, verbose => $verbose); 1570 unless ($success) { 1571 print STDERR " residexp had an issue setting the state:? $success $error_code\n"; 1572 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1573 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1574 } 1575 } else { 1576 my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; 1577 $command .= " -dbname $dbname" if defined $dbname; 1578 1579 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1580 run(command => $command, verbose => $verbose); 1581 unless ($success) { 1582 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1583 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1584 } 1585 exit $PS_EXIT_UNKNOWN_ERROR; 1586 } 1587 } 1588 # Check to see if we can mark the whole detRunSummary object as cleaned. 1589 1590 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1591 $command .= " -dbname $dbname" if defined $dbname; 1592 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1593 unless ($success) { 1594 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1595 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1596 } 1597 if (@$stdout_buf != 0) { 1598 $exps = $mdcParser->parse_list(join "", @$stdout_buf) or 1599 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1600 1601 foreach my $exp (@$exps) { 1602 my $iteration = $exp->{iteration}; 1603 my $command; 1604 if ($mode eq "goto_cleaned") { 1605 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1606 } 1607 if ($mode eq "goto_scrubbed") { 1608 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1609 } 1610 if ($mode eq "goto_purged") { 1611 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1612 } 1613 $command .= " -dbname $dbname" if defined $dbname; 1614 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1615 run(command => $command, verbose => $verbose); 1616 unless ($success) { 1617 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1618 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1619 } 1620 } 1621 } 1622 exit 0; 1623 } 1624 565 1625 566 1626 die "ipp_cleanup.pl -stage $stage not yet implemented\n"; … … 569 1629 { 570 1630 my $files = shift; # reference to a list of files to unlink 1631 # my $test_verbose = 1; 1632 1633 # if ($test_verbose == 1) { 1634 # open(TMPLOG,">>/tmp/czw.cleanup.log"); 1635 # flock(TMPLOG,2); 1636 # } 571 1637 572 1638 # this script is, of course, very dangerous. 573 1639 foreach my $file (@$files) { 574 print STDERR "unlinking $file\n"; 1640 print STDERR "unlinking $stage $stage_id $file"; 1641 unless ($ipprc->file_exists($file)) { 1642 print STDERR "\t File not found\n"; 1643 } 1644 else { 1645 print STDERR "\n"; 1646 } 1647 1648 # if ($test_verbose == 1) { 1649 # print TMPLOG "$stage $stage_id $file"; 1650 1651 # else { 1652 # print TMPLOG "\n"; 1653 # } 1654 # } 1655 575 1656 $ipprc->file_delete($file); 576 1657 } 1658 1659 # if ($test_verbose == 1) { 1660 # flock(TMPLOG,8); 1661 # close(TMPLOG); 1662 # } 1663 577 1664 return 1; 578 1665 }
Note:
See TracChangeset
for help on using the changeset viewer.
