Changeset 27638
- Timestamp:
- Apr 8, 2010, 10:14:37 AM (16 years ago)
- File:
-
- 1 edited
-
trunk/ippScripts/scripts/ipp_cleanup.pl (modified) (45 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/ipp_cleanup.pl
r27144 r27638 61 61 62 62 my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, 63 64 63 "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0, 64 "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 ); 65 65 unless (exists($stages{$stage})) { 66 66 die "unknown stage $stage for ipp_cleanup.pl\n"; … … 96 96 # XXX Why? This could just mean there's nothing to cleanup, or that we're trying to rerun an errored run. 97 97 if (@$stdout_buf == 0) { 98 99 100 101 102 103 104 105 106 107 98 my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state"; 99 $command .= " -dbname $dbname" if defined $dbname; 100 101 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 102 run(command => $command, verbose => $verbose); 103 unless ($success) { 104 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 105 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 106 } 107 exit 0; 108 108 } 109 109 … … 120 120 my $path_base = $imfile->{path_base}; 121 121 my $status = 1; 122 $status = 0 unless defined $path_base and $path_base ne "NULL"; 122 123 123 124 # don't clean up unless the data needed to update is available 124 125 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 125 # goto_scrubbed now requires the config file to not exist. 126 if ($mode eq "goto_cleaned") { 127 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 128 129 unless ($ipprc->file_exists($config_file)) { 130 print STDERR "skipping cleanup for chipRun $stage_id $class_id " 131 . " because config file ($config_file) is missing\n"; 132 $status = 0; 133 } 134 } 135 elsif ($mode eq "goto_scrubbed") { 136 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 137 138 if ($ipprc->file_exists($config_file)) { 139 print STDERR "skipping scrubbed for chipRun $stage_id $class_id " 140 . " because config file ($config_file) is present\n"; 141 $status = 0; 142 } 143 } 126 # goto_scrubbed now requires the config file to not exist. 127 if ($status) { 128 if ($mode eq "goto_cleaned") { 129 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 130 131 unless ($ipprc->file_exists($config_file)) { 132 print STDERR "skipping cleanup for chipRun $stage_id $class_id " 133 . " because config file ($config_file) is missing\n"; 134 $status = 0; 135 } 136 } 137 elsif ($mode eq "goto_scrubbed") { 138 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 139 140 if ($ipprc->file_exists($config_file)) { 141 print STDERR "skipping scrubbed for chipRun $stage_id $class_id " 142 . " because config file ($config_file) is present\n"; 143 $status = 0; 144 } 145 } 146 } 144 147 145 148 if ($status) { … … 175 178 $command .= " -topurgedimfile"; 176 179 } 177 180 elsif ($mode eq "goto_cleaned") { 178 181 $command .= " -tocleanedimfile"; 179 182 } 180 181 182 183 elsif ($mode eq "goto_scrubbed") { 184 $command .= " -toscrubbedimfile"; 185 } 183 186 184 187 $command .= " -dbname $dbname" if defined $dbname; … … 191 194 } 192 195 193 194 195 196 $command .= " -dbname $dbname" if defined $dbname; 197 198 199 200 201 202 196 # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. 197 my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; 198 $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; 199 $command .= " -dbname $dbname" if defined $dbname; 200 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 201 run(command => $command, verbose => $verbose); 202 unless ($success) { 203 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 204 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); 205 } 203 206 204 207 } else { 205 208 206 207 208 209 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) 210 my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -set_state $error_state"; 211 $command .= " -dbname $dbname" if defined $dbname; 209 212 210 213 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 214 217 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 215 218 } 216 217 218 219 220 221 222 223 224 225 226 219 220 # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. 221 $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state"; 222 $command .= " -dbname $dbname" if defined $dbname; 223 224 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 225 run(command => $command, verbose => $verbose); 226 unless ($success) { 227 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 228 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 229 } 227 230 228 231 } … … 264 267 if ($mode eq "goto_cleaned") { 265 268 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); 266 267 269 270 unless ($ipprc->file_exists($config_file)) { 268 271 print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n"; 269 272 $status = 0; … … 271 274 } 272 275 elsif ($mode eq "goto_scrubbed") { 273 274 275 276 277 278 276 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); 277 278 if ($ipprc->file_exists($config_file)) { 279 print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; 280 $status = 0; 281 } 279 282 } 280 283 if ($status) { … … 296 299 if ($mode eq "goto_cleaned") { 297 300 $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned"; 298 301 } 299 302 if ($mode eq "goto_scrubbed") { 300 303 $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed"; 301 304 } 302 305 if ($mode eq "goto_purged") { 303 306 $command = "$camtool -updaterun -cam_id $stage_id -set_state purged"; … … 311 314 } 312 315 313 314 315 316 # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. 317 my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; 318 $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; 316 319 $command .= " -dbname $dbname" if defined $dbname; 317 318 319 320 321 322 320 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 321 run(command => $command, verbose => $verbose); 322 unless ($success) { 323 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 324 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); 325 } 323 326 324 327 } else { 325 328 # since 'camera' has only a single imfile, we can just update the run 326 329 my $command = "$camtool -updaterun -cam_id $stage_id -set_state $error_state"; 327 330 $command .= " -dbname $dbname" if defined $dbname; … … 356 359 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); 357 360 } 358 361 359 362 if (@$stdout_buf == 0) { 360 361 362 363 364 365 366 367 368 369 370 371 372 363 # No skycells were found for some reason. 364 # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." 365 my $command = "$warptool -updaterun -warp_id $stage_id -set_state $error_state"; 366 $command .= " -dbname $dbname" if defined $dbname; 367 368 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 369 run(command => $command, verbose => $verbose); 370 unless ($success) { 371 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 372 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); 373 } 374 375 exit(0); 373 376 } 374 377 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or … … 384 387 385 388 my $status = 1; 386 if ($mode eq "goto_cleaned") { 387 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 388 389 unless ($ipprc->file_exists($config_file)) { 390 print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . 391 " because config file is missing\n"; 392 $status = 0; 393 } 394 } 395 elsif ($mode eq "goto_scrubbed") { 396 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 397 398 if ($ipprc->file_exists($config_file)) { 399 print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . 400 " because config file is present\n"; 401 $status = 0; 402 } 403 } 404 if ($status) { 405 if ($skyfile->{quality} != 8007) { 406 my @files = (); 407 408 # delete the temporary image datafiles 409 addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); 410 addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); 411 addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); 389 $status = 0 unless defined $path_base and $path_base ne "NULL"; 390 391 if ($status) { 392 if ($mode eq "goto_cleaned") { 393 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 394 395 unless ($ipprc->file_exists($config_file)) { 396 print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . 397 " because config file is missing\n"; 398 $status = 0; 399 } 400 } 401 elsif ($mode eq "goto_scrubbed") { 402 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 403 404 if ($ipprc->file_exists($config_file)) { 405 print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . 406 " because config file is present\n"; 407 $status = 0; 408 } 409 } 410 } 411 412 if ($status) { 413 if ($skyfile->{quality} != 8007) { 414 my @files = (); 415 416 # delete the temporary image datafiles 417 addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); 418 addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); 419 addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); 412 420 # addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); 413 414 415 416 417 418 419 420 421 422 423 421 if ($mode eq "goto_purged") { 422 # additional files to remove for 'purge' mode 423 addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); 424 addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); 425 addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); 426 # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); 427 428 # XXX: do we want to delete these? 429 # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); 430 # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); 431 } 424 432 # actual command to delete the files 425 426 433 $status = &delete_files (\@files); 434 } 427 435 } 428 436 … … 431 439 if ($mode eq "goto_purged") { 432 440 $command .= " -topurgedskyfile"; 433 } 434 441 } 442 elsif ($mode eq "goto_cleaned") { 435 443 $command .= " -tocleanedskyfile"; 436 444 } 437 438 439 445 elsif ($mode eq "goto_scrubbed") { 446 $command .= " -toscrubbedskyfile"; 447 } 440 448 $command .= " -dbname $dbname" if defined $dbname; 441 449 … … 447 455 } 448 456 449 450 451 452 $command .= " -dbname $dbname" if defined $dbname; 453 454 455 456 457 458 457 # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. 458 my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; 459 $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; 460 $command .= " -dbname $dbname" if defined $dbname; 461 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 462 run(command => $command, verbose => $verbose); 463 unless ($success) { 464 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 465 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); 466 } 459 467 460 468 } else { 461 462 469 my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; 470 $command .= " -dbname $dbname" if defined $dbname; 463 471 464 472 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 469 477 } 470 478 471 479 # exit $PS_EXIT_UNKNOWN_ERROR; 472 480 } 473 481 } … … 487 495 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 488 496 unless ($success) { 489 490 497 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 498 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 491 499 } 492 500 493 501 if (@$stdout_buf == 0) { 494 495 496 497 498 499 500 501 502 503 504 505 506 507 } 508 509 my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 510 511 512 $skyfiles = parse_md_list($metadata) or 513 514 502 # No skycells were found for some reason. 503 # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." 504 my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state"; 505 $command .= " -dbname $dbname" if defined $dbname; 506 507 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 508 run(command => $command, verbose => $verbose); 509 unless ($success) { 510 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 511 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 512 } 513 514 exit(0); 515 } 516 517 my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 518 &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR); 519 520 $skyfiles = parse_md_list($metadata) or 521 &my_die("Unable to parse metadata list", "stack", $stage_id, $PS_EXIT_PROG_ERROR); 522 515 523 my @files = (); 516 524 foreach my $skyfile (@{ $skyfiles }) { 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 } 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 $command .= " -dbname $dbname" if defined $dbname; 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 # 608 525 my $path_base = $skyfile->{path_base}; 526 my $skycell_id = $skyfile->{skycell_id}; 527 528 my $status = 1; 529 if ((!exists($skyfile->{path_base}))|| 530 (!defined($path_base))) { 531 $status = 0; 532 } 533 if ($status) { 534 if ($mode eq "goto_cleaned") { 535 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); 536 537 unless ($ipprc->file_exists($config_file)) { 538 print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" . 539 " because config file is missing\n"; 540 $status = 0; 541 } 542 } 543 elsif ($mode eq "goto_scrubbed") { 544 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); 545 546 if ($ipprc->file_exists($config_file)) { 547 print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . 548 " because config file is present\n"; 549 $status = 0; 550 } 551 } 552 } 553 554 if ($status) { 555 my @files = (); 556 # delete the temporary image datafiles 557 addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id); 558 addFilename(\@files, "PPSTACK.OUTPUT.MASK", $path_base, $skycell_id); 559 addFilename(\@files, "PPSTACK.OUTPUT.VARIANCE", $path_base, $skycell_id); 560 561 if ($mode eq "goto_purged") { 562 # additional files to remove for 'purge' mode 563 addFilename(\@files, "PPSTACK.CONV.KERNEL", $path_base, $skycell_id); 564 addFilename(\@files, "PPSTACK.OUTPUT.JPEG1", $path_base, $skycell_id); 565 addFilename(\@files, "PPSTACK.OUTPUT.JPEG2", $path_base, $skycell_id); 566 # Commented out to match warp files. 567 #addFilename(\@files, "PPSTACK.TARGET.PSF", $path_base, $skycell_id); 568 #addFilename(\@files, "PPSTACK.CONFIG", $path_base, $skycell_id); 569 } 570 571 $status = &delete_files(\@files); 572 } 573 574 if ($status) { 575 my $command = "$stacktool -stack_id $stage_id"; 576 if ($mode eq "goto_purged") { 577 $command .= " -updaterun -set_state purged"; 578 } 579 elsif ($mode eq "goto_cleaned") { 580 $command .= " -updaterun -set_state cleaned"; 581 } 582 elsif ($mode eq "goto_scrubbed") { 583 $command .= " -updaterun -set_state scrubbed"; 584 } 585 $command .= " -dbname $dbname" if defined $dbname; 586 587 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 588 run(command => $command, verbose => $verbose); 589 unless ($success) { 590 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 591 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 592 } 593 594 # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. 595 my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; 596 $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; 597 $command .= " -dbname $dbname" if defined $dbname; 598 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 599 run(command => $command, verbose => $verbose); 600 unless ($success) { 601 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 602 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); 603 } 604 605 } else { 606 my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state"; 607 $command .= " -dbname $dbname" if defined $dbname; 608 609 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 610 run(command => $command, verbose => $verbose); 611 unless ($success) { 612 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 613 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 614 } 615 # exit $PS_EXIT_UNKNOWN_ERROR; 616 } 609 617 } 610 618 exit 0; … … 612 620 613 621 if ($stage eq 'diff') { 614 622 615 623 die "--stage_id required for stage diff\n" if !$stage_id; 616 624 … … 624 632 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 625 633 unless ($success) { 626 627 634 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 635 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 628 636 } 629 637 630 638 if (@$stdout_buf == 0) { 631 632 633 634 635 636 637 638 639 640 641 642 643 644 } 645 646 my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 647 648 649 $skyfiles = parse_md_list($metadata) or 650 651 639 # No skycells were found for some reason. 640 # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." 641 my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; 642 $command .= " -dbname $dbname" if defined $dbname; 643 644 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 645 run(command => $command, verbose => $verbose); 646 unless ($success) { 647 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 648 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 649 } 650 651 exit(0); 652 } 653 654 my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 655 &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); 656 657 $skyfiles = parse_md_list($metadata) or 658 &my_die("Unable to parse metadata list", "diff", $stage_id, $PS_EXIT_PROG_ERROR); 659 652 660 my @files = (); 653 661 foreach my $skyfile (@{ $skyfiles }) { 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 # 714 715 716 # 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 $command .= " -dbname $dbname" if defined $dbname; 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 # 762 662 my $path_base = $skyfile->{path_base}; 663 my $skycell_id = $skyfile->{skycell_id}; 664 665 my $status = 1; 666 if ((!exists($skyfile->{path_base}))|| 667 (!defined($path_base))) { 668 $status = 0; 669 } 670 if ($status) { 671 672 if ($mode eq "goto_cleaned") { 673 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); 674 675 unless ($ipprc->file_exists($config_file)) { 676 print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" . 677 " because config file ($config_file) is missing\n"; 678 $status = 0; 679 } 680 } 681 elsif ($mode eq "goto_scrubbed") { 682 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); 683 684 if ($ipprc->file_exists($config_file)) { 685 print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . 686 " because config file ($config_file) is present\n"; 687 $status = 0; 688 } 689 } 690 } 691 if ($status) { 692 my @files = (); 693 # delete the temporary image datafiles 694 addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id); 695 addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id); 696 addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id); 697 698 addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id); 699 addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id); 700 addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id); 701 702 addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id); 703 addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id); 704 addFilename(\@files, "PPSUB.INPUT.CONV.VARIANCE", $path_base, $skycell_id); 705 706 addFilename(\@files, "PPSUB.REF.CONV", $path_base, $skycell_id); 707 addFilename(\@files, "PPSUB.REF.CONV.MASK", $path_base, $skycell_id); 708 addFilename(\@files, "PPSUB.REF.CONV.VARIANCE", $path_base, $skycell_id); 709 710 if ($mode eq "goto_purged") { 711 # additional files to remove for 'purge' mode 712 addFilename(\@files, "PPSUB.OUTPUT.KERNELS", $path_base, $skycell_id); 713 addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id); 714 addFilename(\@files, "PPSUB.OUTPUT.JPEG2", $path_base, $skycell_id); 715 # Commented out to match warp files. 716 #addFilename(\@files, "PPSUB.CONFIG", $path_base, $skycell_id); 717 addFilename(\@files, "PPSUB.OUTPUT.SOURCES", $path_base, $skycell_id); 718 addFilename(\@files, "PPSUB.INVERSE.SOURCES", $path_base, $skycell_id); 719 720 } 721 # print STDERR "MY FILES: @files\n"; 722 $status = &delete_files(\@files); 723 } 724 # print STDERR "MY STATUS: $status\n"; 725 if ($status) { 726 my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; 727 728 if ($mode eq "goto_purged") { 729 $command .= " -topurgedskyfile"; 730 } 731 elsif ($mode eq "goto_cleaned") { 732 $command .= " -tocleanedskyfile"; 733 } 734 elsif ($mode eq "goto_scrubbed") { 735 $command .= " -toscrubbedskyfile"; 736 } 737 738 $command .= " -dbname $dbname" if defined $dbname; 739 740 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 741 run(command => $command, verbose => $verbose); 742 unless ($success) { 743 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 744 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 745 } 746 747 # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. 748 my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; 749 $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; 750 $command .= " -dbname $dbname" if defined $dbname; 751 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 752 run(command => $command, verbose => $verbose); 753 unless ($success) { 754 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 755 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); 756 } 757 758 } else { 759 my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; 760 761 $command .= " -dbname $dbname" if defined $dbname; 762 763 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 764 run(command => $command, verbose => $verbose); 765 unless ($success) { 766 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 767 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 768 } 769 # exit $PS_EXIT_UNKNOWN_ERROR; 770 } 763 771 } 764 772 exit 0; … … 786 794 # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' 787 795 if (@$stdout_buf == 0) { 788 789 790 791 792 793 794 795 796 797 796 my $command = "$faketool -fake_id $stage_id -updaterun -set_state $error_state"; 797 $command .= " -dbname $dbname" if defined $dbname; 798 799 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 800 run(command => $command, verbose => $verbose); 801 unless ($success) { 802 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 803 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 804 } 805 exit 0; 798 806 } 799 807 … … 813 821 # don't clean up unless the data needed to update is available 814 822 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 815 823 # goto_scrubbed now requires the config file to not exist. 816 824 if ($mode eq "goto_cleaned") { 817 825 my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); 818 826 819 827 unless ($ipprc->file_exists($config_file)) { 820 828 print STDERR "skipping cleanup for fakeRun $stage_id $class_id " 821 829 . " because config file is missing\n"; … … 823 831 } 824 832 } 825 826 827 828 829 830 831 832 833 833 elsif ($mode eq "goto_scrubbed") { 834 my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); 835 836 if ($ipprc->file_exists($config_file)) { 837 print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " 838 . " because config file is present\n"; 839 $status = 0; 840 } 841 } 834 842 835 843 if ($status) { … … 844 852 if ($mode eq "goto_purged") { 845 853 # additional files to remove for 'purge' mode 846 847 848 854 addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id); 855 addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id); 856 addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id); 849 857 } 850 858 … … 858 866 $command .= " -topurgedimfile"; 859 867 } 860 868 elsif ($mode eq "goto_cleaned") { 861 869 $command .= " -tocleanedimfile"; 862 870 } 863 864 865 871 elsif ($mode eq "goto_scrubbed") { 872 $command .= " -toscrubbedimfile"; 873 } 866 874 867 875 $command .= " -dbname $dbname" if defined $dbname; … … 875 883 } else { 876 884 877 878 879 885 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) 886 my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state"; 887 $command .= " -dbname $dbname" if defined $dbname; 880 888 881 889 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 889 897 exit 0; 890 898 891 } 899 } 892 900 # Detrend stages 893 901 if ($stage eq "detrend.processed") { … … 916 924 # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' 917 925 if (@$stdout_buf != 0) { 918 # 919 920 921 926 # exit 0; # Silently exit if there's nothing to do. I don't know how we'd ever get here, but let's be safe. 927 928 929 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 922 930 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 923 924 925 926 931 932 # extract the metadata for the files into a hash list 933 $imfiles = parse_md_list($metadata) or 934 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 927 935 } 928 936 # loop over all of the imfiles, determine the path_base and class_id for each 929 937 foreach my $imfile (@$imfiles) { 930 938 my $iexp_id = $imfile->{exp_id}; 931 939 my $class_id = $imfile->{class_id}; 932 940 my $path_base = $imfile->{path_base}; 933 941 my $status = 1; 934 942 935 936 937 938 939 940 941 942 943 # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent, 944 943 unless (defined($path_base)) { 944 print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n"; 945 $status = 0; 946 } 947 unless (defined($class_id)) { 948 print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n"; 949 $status = 0; 950 } 951 # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent, 952 # and so there is no check for config files. 945 953 if ($status) { 946 954 # array of actual filenames to delete … … 952 960 if ($mode eq "goto_purged") { 953 961 # additional files to remove for 'purge' mode 954 955 962 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 963 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 956 964 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 957 965 } … … 965 973 $command .= " -data_state purged"; 966 974 } 967 975 elsif ($mode eq "goto_cleaned") { 968 976 $command .= " -data_state cleaned"; 969 977 } 970 971 972 978 elsif ($mode eq "goto_scrubbed") { 979 $command .= " -data_state scrubbed"; 980 } 973 981 $command .= " -dbname $dbname" if defined $dbname; 974 982 … … 981 989 982 990 } else { 983 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* 984 985 986 991 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* 992 my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile "; 993 $command .= " -data_state $error_state"; 994 $command .= " -dbname $dbname" if defined $dbname; 987 995 988 996 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 992 1000 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 993 1001 } 994 1002 } 995 1003 } 996 1004 … … 1001 1009 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1002 1010 unless ($success) { 1003 1004 1011 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1012 &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code); 1005 1013 } 1006 1014 if (@$stdout_buf != 0) { 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 # 1026 1027 1028 1029 1030 1031 1032 1033 } 1034 1015 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1016 &my_die("Unable to parse metadata config doc", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); 1017 my $exps = parse_md_list($metadata) or 1018 &my_die("Unable to parse metadata list", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); 1019 1020 foreach my $exp (@$exps) { 1021 my $exp_id = $exp->{exp_id}; 1022 my $command = "$dettool -updateprocessedexp -det_id $det_id -exp_id $exp_id "; 1023 if ($mode eq "goto_cleaned") { 1024 $command .= " -data_state cleaned "; 1025 } 1026 if ($mode eq "goto_scrubbed") { 1027 $command .= " -data_state scrubbed "; 1028 } 1029 if ($mode eq "goto_purged") { 1030 $command .= " -data_state purged "; 1031 } 1032 $command .= " -dbname $dbname" if defined $dbname; 1033 # print "$command\n"; 1034 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1035 run(command => $command, verbose => $verbose); 1036 unless ($success) { 1037 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1038 &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code); 1039 } 1040 } 1041 } 1042 1035 1043 exit 0; 1036 1044 } … … 1057 1065 &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); 1058 1066 } 1059 1067 1060 1068 # if there are no detResidImfiles (@$stdout_buf == 0), then silently exit. 1061 1069 if (@$stdout_buf != 0) { 1062 1063 1064 1065 1066 1067 1070 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1071 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1072 1073 # extract the metadata for the files into a hash list 1074 $imfiles = parse_md_list($metadata) or 1075 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1068 1076 } 1069 1077 # loop over all of the imfiles, determine the path_base and class_id for each 1070 1078 foreach my $imfile (@$imfiles) { 1071 1079 my $iexp_id = $imfile->{exp_id}; 1072 1080 my $class_id = $imfile->{class_id}; 1073 1081 my $path_base = $imfile->{path_base}; 1074 1082 my $iteration = $imfile->{iteration}; 1075 1083 1076 1084 my $status = 1; 1077 1085 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1086 # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent, 1087 # and so there is no check for config files. 1088 unless (defined($path_base)) { 1089 print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n"; 1090 $status = 0; 1091 } 1092 unless (defined($class_id)) { 1093 print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n"; 1094 $status = 0; 1095 } 1088 1096 if ($status) { 1089 1097 # array of actual filenames to delete … … 1094 1102 if ($mode eq "goto_purged") { 1095 1103 # additional files to remove for 'purge' mode 1096 1097 1104 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 1105 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 1098 1106 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 1099 1107 } 1100 # 1101 # 1102 # 1108 # foreach my $f (@files) { 1109 # print "RESID: $f\n"; 1110 # } 1103 1111 # actual command to delete the files 1104 1112 $status = &delete_files (\@files); … … 1110 1118 $command .= " -data_state purged"; 1111 1119 } 1112 1120 elsif ($mode eq "goto_cleaned") { 1113 1121 $command .= " -data_state cleaned"; 1114 1122 } 1115 1116 1117 1123 elsif ($mode eq "goto_scrubbed") { 1124 $command .= " -data_state scrubbed"; 1125 } 1118 1126 1119 1127 $command .= " -dbname $dbname" if defined $dbname; … … 1126 1134 } 1127 1135 } else { 1128 1129 1130 1136 my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile "; 1137 $command .= " -data_state $error_state "; 1138 $command .= " -dbname $dbname" if defined $dbname; 1131 1139 1132 1140 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 1140 1148 # Flag the detResidExp as clean now as well, if it is marked tobe cleaned (this is still clunky). 1141 1149 1142 $command = "$dettool -pendingcleanup_residexp -det_id $det_id -exp_id $exp_id"; 1150 $command = "$dettool -pendingcleanup_residexp -det_id $det_id -exp_id $exp_id"; 1143 1151 $command .= " -dbname $dbname" if defined $dbname; 1144 1152 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1145 1153 unless ($success) { 1146 1147 1154 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1155 &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code); 1148 1156 } 1149 1157 if (@$stdout_buf != 0) { 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1158 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1159 &my_die("Unable to parse metadata config doc", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); 1160 my $exps = parse_md_list($metadata) or 1161 &my_die("Unable to parse metadata list", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); 1162 1163 foreach my $exp (@$exps) { 1164 my $iteration = $exp->{iteration}; 1165 my $command = "$dettool -updateresidexp -det_id $det_id -exp_id $exp_id "; 1166 if ($mode eq "goto_cleaned") { 1167 $command .= " -data_state cleaned"; 1168 } 1169 if ($mode eq "goto_scrubbed") { 1170 $command .= " -data_state scrubbed"; 1171 } 1172 if ($mode eq "goto_purged") { 1173 $command .= " -data_state purged"; 1174 } 1175 $command .= " -dbname $dbname" if defined $dbname; 1176 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1177 run(command => $command, verbose => $verbose); 1178 unless ($success) { 1179 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1180 &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code); 1181 } 1182 } 1175 1183 } 1176 1184 … … 1179 1187 1180 1188 if ($stage eq "detrend.stack.imfile") { 1181 1189 1182 1190 die "--stage_id required for stage $stage\n" if !$stage_id; 1183 1191 … … 1191 1199 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1192 1200 unless ($success) { 1193 1194 1195 } 1196 my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 1197 1198 1199 $stacks = parse_md_list($metadata) or 1200 1201 1201 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1202 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1203 } 1204 my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 1205 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1206 1207 $stacks = parse_md_list($metadata) or 1208 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1209 1202 1210 my @files = (); 1203 1211 foreach my $stack (@{ $stacks }) { 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 # 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 } 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1212 # detStackedImfile does not have a path_base column. This is inconvenient, as it means we need to calculate it. 1213 my $path_base = $stack->{uri}; 1214 my $iteration = $stack->{iteration}; 1215 my $class_id = $stack->{class_id}; 1216 1217 $path_base =~ s/\.fits$//; # That should do it? 1218 1219 my $status = 1; 1220 1221 if ($status) { 1222 my @files = (); 1223 # delete the temporary image datafiles 1224 # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. 1225 # I understand that they all point to the same filename right now, but that may not be true in 1226 # the future. 1227 addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id); 1228 addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id); 1229 addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id); 1230 addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id); 1231 addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id); 1232 addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id); 1233 1234 1235 addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id); 1236 addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id); 1237 1238 if ($mode eq "goto_purged") { 1239 # additional files to remove for 'purge' mode 1240 # addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id); 1241 } 1242 1243 $status = &delete_files(\@files); 1244 } 1245 1246 if ($status) { 1247 my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id"; 1248 if ($mode eq "goto_purged") { 1249 $command .= " -updatestacked -data_state purged"; 1250 } 1251 elsif ($mode eq "goto_cleaned") { 1252 $command .= " -updatestacked -data_state cleaned"; 1253 } 1254 elsif ($mode eq "goto_scrubbed") { 1255 $command .= " -updatestacked -data_state scrubbed"; 1256 } 1257 $command .= " -dbname $dbname" if defined $dbname; 1258 1259 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1260 run(command => $command, verbose => $verbose); 1261 unless ($success) { 1262 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1263 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1264 } 1265 } else { 1266 my $command = "$dettool -updatestacked -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1267 $command .= " -dbname $dbname" if defined $dbname; 1268 1269 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1270 run(command => $command, verbose => $verbose); 1271 unless ($success) { 1272 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1273 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1274 } 1275 exit $PS_EXIT_UNKNOWN_ERROR; 1276 } 1269 1277 } 1270 1278 # Check to see if we can mark the whole detRunSummary object as cleaned. 1271 1279 1272 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1280 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1273 1281 $command .= " -dbname $dbname" if defined $dbname; 1274 1282 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1275 1283 unless ($success) { 1276 1277 1284 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1285 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1278 1286 } 1279 1287 if (@$stdout_buf != 0) { 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1288 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1289 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1290 my $exps = parse_md_list($metadata) or 1291 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1292 1293 foreach my $exp (@$exps) { 1294 my $iteration = $exp->{iteration}; 1295 my $command; 1296 if ($mode eq "goto_cleaned") { 1297 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1298 } 1299 if ($mode eq "goto_scrubbed") { 1300 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1301 } 1302 if ($mode eq "goto_purged") { 1303 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1304 } 1305 $command .= " -dbname $dbname" if defined $dbname; 1306 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1307 run(command => $command, verbose => $verbose); 1308 unless ($success) { 1309 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1310 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1311 } 1312 } 1305 1313 } 1306 1314 exit 0; … … 1326 1334 1327 1335 foreach my $exp (@$exps) { 1328 # 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1336 # my $path_base = $exp->{path_base}; 1337 my $iteration = $exp->{iteration}; 1338 my $class_id = $exp->{class_id}; 1339 1340 my $status = 1; 1341 if ($status) { 1342 my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id"; 1343 if ($mode eq "goto_cleaned") { 1344 $command .= " -data_state cleaned"; 1345 } 1346 if ($mode eq "goto_scrubbed") { 1347 $command .= " -data_state scrubbed"; 1348 } 1349 if ($mode eq "goto_purged") { 1350 $command .= " -data_state purged"; 1351 } 1352 $command .= " -dbname $dbname" if defined $dbname; 1353 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1354 run(command => $command, verbose => $verbose); 1355 unless ($success) { 1356 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1357 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1358 } 1359 } else { 1360 my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1361 $command .= " -dbname $dbname" if defined $dbname; 1362 1363 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1364 run(command => $command, verbose => $verbose); 1365 unless ($success) { 1366 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1367 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1368 } 1369 exit $PS_EXIT_UNKNOWN_ERROR; 1370 } 1363 1371 } 1364 1372 # Check to see if we can mark the whole detRunSummary object as cleaned. 1365 1373 1366 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1374 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1367 1375 $command .= " -dbname $dbname" if defined $dbname; 1368 1376 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1369 1377 unless ($success) { 1370 1371 1378 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1379 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1372 1380 } 1373 1381 if (@$stdout_buf != 0) { 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1382 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1383 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1384 $exps = parse_md_list($metadata) or 1385 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1386 1387 foreach my $exp (@$exps) { 1388 my $iteration = $exp->{iteration}; 1389 my $command; 1390 if ($mode eq "goto_cleaned") { 1391 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1392 } 1393 if ($mode eq "goto_scrubbed") { 1394 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1395 } 1396 if ($mode eq "goto_purged") { 1397 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1398 } 1399 $command .= " -dbname $dbname" if defined $dbname; 1400 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1401 run(command => $command, verbose => $verbose); 1402 unless ($success) { 1403 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1404 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1405 } 1406 } 1399 1407 } 1400 1408 … … 1424 1432 1425 1433 foreach my $exp (@$exps) { 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1434 my $path_base = $exp->{path_base}; 1435 my $iteration = $exp->{iteration}; 1436 my $class_id = $exp->{class_id}; 1437 1438 my $status = 1; 1439 # don't clean up unless the data needed to update is available 1440 # goto_scrubbed now requires the config file to not be present 1441 if ($mode eq "goto_cleaned") { 1442 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1443 1444 unless ($ipprc->file_exists($config_file)) { 1445 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 1446 $status = 0; 1447 } 1448 } 1449 elsif ($mode eq "goto_scrubbed") { 1450 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1451 1452 if ($ipprc->file_exists($config_file)) { 1453 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 1454 $status = 0; 1455 } 1456 } 1457 if ($status) { 1458 my @files = (); 1459 1460 if ($mode eq "goto_purged") { 1461 # additional files to remove for 'purge' mode 1462 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base); 1463 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base); 1464 1465 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base); 1466 addFilename (\@files, "PPIMAGE.STATS", $path_base); 1467 } 1468 # actual command to delete the files 1469 $status = &delete_files (\@files); 1470 } 1471 1472 if ($status) { 1473 my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id"; 1474 if ($mode eq "goto_cleaned") { 1475 $command .= " -data_state cleaned"; 1476 } 1477 if ($mode eq "goto_scrubbed") { 1478 $command .= " -data_state scrubbed"; 1479 } 1480 if ($mode eq "goto_purged") { 1481 $command .= " -data_state purged"; 1482 } 1483 $command .= " -dbname $dbname" if defined $dbname; 1484 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1485 run(command => $command, verbose => $verbose); 1486 unless ($success) { 1487 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1488 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1489 } 1490 } else { 1491 my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1492 $command .= " -dbname $dbname" if defined $dbname; 1493 1494 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1495 run(command => $command, verbose => $verbose); 1496 unless ($success) { 1497 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1498 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1499 } 1500 exit $PS_EXIT_UNKNOWN_ERROR; 1501 } 1494 1502 } 1495 1503 # Check to see if we can mark the whole detRunSummary object as cleaned. 1496 1504 1497 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1505 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1498 1506 $command .= " -dbname $dbname" if defined $dbname; 1499 1507 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1500 1508 unless ($success) { 1501 1502 1509 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1510 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1503 1511 } 1504 1512 if (@$stdout_buf != 0) { 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1513 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1514 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1515 $exps = parse_md_list($metadata) or 1516 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1517 1518 foreach my $exp (@$exps) { 1519 my $iteration = $exp->{iteration}; 1520 my $command; 1521 if ($mode eq "goto_cleaned") { 1522 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1523 } 1524 if ($mode eq "goto_scrubbed") { 1525 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1526 } 1527 if ($mode eq "goto_purged") { 1528 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1529 } 1530 $command .= " -dbname $dbname" if defined $dbname; 1531 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1532 run(command => $command, verbose => $verbose); 1533 unless ($success) { 1534 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1535 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1536 } 1537 } 1530 1538 } 1531 1539 … … 1549 1557 1550 1558 if (@$stdout_buf == 0) { 1551 1559 exit 0; 1552 1560 } 1553 1561 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or … … 1558 1566 1559 1567 foreach my $exp (@$exps) { 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1568 my $exp_id = $exp->{exp_id}; 1569 my $iteration = $exp->{iteration}; 1570 my $path_base = $exp->{path_base}; 1571 1572 my $status = 1; 1573 # don't clean up unless the data needed to update is available 1574 # goto_scrubbed now requires the config file to not be present 1575 if ($mode eq "goto_cleaned") { 1576 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1577 1578 unless ($ipprc->file_exists($config_file)) { 1579 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 1580 $status = 0; 1581 } 1582 } 1583 elsif ($mode eq "goto_scrubbed") { 1584 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1585 1586 if ($ipprc->file_exists($config_file)) { 1587 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 1588 $status = 0; 1589 } 1590 } 1591 if ($status) { 1592 my @files = (); 1593 # delete the temporary image datafiles 1594 if ($mode eq "goto_purged") { 1595 # additional files to remove for 'purge' mode 1596 addFilename (\@files, "PPIMAGE.JPEG1", $path_base); 1597 addFilename (\@files, "PPIMAGE.JPEG2", $path_base); 1598 } 1599 # actual command to delete the files 1600 $status = &delete_files (\@files); 1601 } 1602 1603 if ($status) { 1604 my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration"; 1605 if ($mode eq "goto_cleaned") { 1606 $command .= " -data_state cleaned"; 1607 } 1608 if ($mode eq "goto_scrubbed") { 1609 $command .= " -data_state scrubbed"; 1610 } 1611 if ($mode eq "goto_purged") { 1612 $command .= " -data_state purged"; 1613 } 1614 $command .= " -dbname $dbname" if defined $dbname; 1615 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1616 run(command => $command, verbose => $verbose); 1617 unless ($success) { 1618 print STDERR " residexp had an issue setting the state:? $success $error_code\n"; 1619 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1620 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1621 } 1622 } else { 1623 my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; 1624 $command .= " -dbname $dbname" if defined $dbname; 1625 1626 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1627 run(command => $command, verbose => $verbose); 1628 unless ($success) { 1629 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1630 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1631 } 1632 exit $PS_EXIT_UNKNOWN_ERROR; 1633 } 1626 1634 } 1627 1635 # Check to see if we can mark the whole detRunSummary object as cleaned. 1628 1636 1629 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1637 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1630 1638 $command .= " -dbname $dbname" if defined $dbname; 1631 1639 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1632 1640 unless ($success) { 1633 1634 1641 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1642 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1635 1643 } 1636 1644 if (@$stdout_buf != 0) { 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1645 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1646 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1647 $exps = parse_md_list($metadata) or 1648 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1649 1650 foreach my $exp (@$exps) { 1651 my $iteration = $exp->{iteration}; 1652 my $command; 1653 if ($mode eq "goto_cleaned") { 1654 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1655 } 1656 if ($mode eq "goto_scrubbed") { 1657 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1658 } 1659 if ($mode eq "goto_purged") { 1660 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1661 } 1662 $command .= " -dbname $dbname" if defined $dbname; 1663 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1664 run(command => $command, verbose => $verbose); 1665 unless ($success) { 1666 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1667 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1668 } 1669 } 1662 1670 } 1663 1671 exit 0; … … 1671 1679 my $files = shift; # reference to a list of files to unlink 1672 1680 # my $test_verbose = 1; 1673 1681 1674 1682 # if ($test_verbose == 1) { 1675 # 1676 # 1683 # open(TMPLOG,">>/tmp/czw.cleanup.log"); 1684 # flock(TMPLOG,2); 1677 1685 # } 1678 1686 … … 1680 1688 foreach my $file (@$files) { 1681 1689 print STDERR "unlinking $stage $stage_id $file"; 1682 1683 1684 1685 1686 1687 1688 1689 # 1690 # 1691 1692 # 1693 # 1694 # 1695 # 1690 unless ($ipprc->file_exists($file)) { 1691 print STDERR "\t File not found\n"; 1692 } 1693 else { 1694 print STDERR "\n"; 1695 } 1696 1697 # if ($test_verbose == 1) { 1698 # print TMPLOG "$stage $stage_id $file"; 1699 1700 # else { 1701 # print TMPLOG "\n"; 1702 # } 1703 # } 1696 1704 1697 1705 $ipprc->file_delete($file); … … 1699 1707 1700 1708 # if ($test_verbose == 1) { 1701 # 1702 # 1709 # flock(TMPLOG,8); 1710 # close(TMPLOG); 1703 1711 # } 1704 1712
Note:
See TracChangeset
for help on using the changeset viewer.
