- Timestamp:
- Jul 10, 2009, 5:53:18 PM (17 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/czw_branch/cleanup/ippScripts/scripts/ipp_cleanup.pl
r24642 r24743 60 60 61 61 62 my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff=> 1,63 detrend.process.imfile => 1, detrend.process.exp => 1, detrend.stack.imfile=> 1,64 detrend.normstat.imfile => 1, detrend.norm.imfile => 1, detrend.norm.exp=> 1,65 detrend.resid.imfile => 1, detrend.resid.exp=> 1 );62 my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, 63 "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1, 64 "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1, 65 "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 ); 66 66 unless ($stages{$stage}) { 67 67 die "unknown stage $stage for ipp_cleanup.pl\n"; … … 119 119 # don't clean up unless the data needed to update is available 120 120 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 121 # goto_scrubbed now requires the config file to not exist. 121 122 if ($mode eq "goto_cleaned") { 122 123 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 123 print STDERR "CHIP: CONFIG_FILE : $config_file\n"; 124 124 125 if (!$config_file or ! -e $config_file) { 125 126 print STDERR "skipping cleanup for chipRun $stage_id $class_id " … … 128 129 } 129 130 } 131 elsif ($mode eq "goto_scrubbed") { 132 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 133 134 if ($config_file and -e $config_file) { 135 print STDERR "skipping scrubbed for chipRun $stage_id $class_id " 136 . " because config file is present\n"; 137 $status = 0; 138 } 139 } 130 140 131 141 if ($status) { … … 160 170 if ($mode eq "goto_purged") { 161 171 $command .= " -topurgedimfile"; 162 } else { 172 } 173 elsif ($mode eq "goto_cleaned") { 163 174 $command .= " -tocleanedimfile"; 164 175 } 176 elsif ($mode eq "goto_scrubbed") { 177 $command .= " -toscrubbedimfile"; 178 } 179 165 180 $command .= " -dbname $dbname" if defined $dbname; 166 181 … … 218 233 my $status = 1; 219 234 # don't clean up unless the data needed to update is available 235 # goto_scrubbed now requires the config file to not be present 220 236 if ($mode eq "goto_cleaned") { 221 237 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); … … 225 241 $status = 0; 226 242 } 243 } 244 elsif ($mode eq "goto_scrubbed") { 245 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); 246 247 if ($config_file and -e $config_file) { 248 print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; 249 $status = 0; 250 } 227 251 } 228 252 if ($status) { … … 246 270 } 247 271 if ($mode eq "goto_scrubbed") { 248 $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";272 $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed"; 249 273 } 250 274 if ($mode eq "goto_purged") { … … 311 335 } 312 336 } 337 elsif ($mode eq "goto_scrubbed") { 338 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 339 340 if ($config_file and -e $config_file) { 341 print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . 342 " because config file is present\n"; 343 $status = 0; 344 } 345 } 313 346 if ($status) { 314 347 # delete the temporary image datafiles … … 337 370 if ($mode eq "goto_purged") { 338 371 $command .= " -topurgedskyfile"; 339 } else { 372 } 373 elsif ($mode eq "goto_cleaned") { 340 374 $command .= " -tocleanedskyfile"; 341 375 } 376 elsif ($mode eq "goto_scrubbed") { 377 $command .= " -toscrubbedskyfile"; 378 } 342 379 $command .= " -dbname $dbname" if defined $dbname; 343 380 … … 395 432 if ($mode eq "goto_cleaned") { 396 433 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); 397 print STDERR "MY CONFIG FILE = $config_file\n"; 398 printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); 434 399 435 $config_file =~ s%^file://%%; 400 436 if (!$config_file or ! -e $config_file) { … … 404 440 } 405 441 $config_file = 'file://' . $config_file; 442 } 443 elsif ($mode eq "goto_scrubbed") { 444 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); 445 $config_file =~ s%^file://%%; 446 if ($config_file and -e $config_file) { 447 print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . 448 " because config file is present\n"; 449 $status = 0; 450 } 406 451 } 407 452 if ($status) { … … 428 473 if ($mode eq "goto_purged") { 429 474 $command .= " -updaterun -state purged"; 430 } else { 475 } 476 elsif ($mode eq "goto_cleaned") { 431 477 $command .= " -updaterun -state cleaned"; 478 } 479 elsif ($mode eq "goto_scrubbed") { 480 $command .= " -updaterun -state scrubbed"; 432 481 } 433 482 $command .= " -dbname $dbname" if defined $dbname; … … 485 534 if ($mode eq "goto_cleaned") { 486 535 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); 487 print STDERR "MY CONFIG FILE = $config_file\n"; 488 printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); 536 489 537 $config_file =~ s%^file://%%; 490 538 if (!$config_file or ! -e $config_file) { … … 494 542 } 495 543 $config_file = 'file://' . $config_file; 544 } 545 elsif ($mode eq "goto_scrubbed") { 546 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); 547 $config_file =~ s%^file://%%; 548 if ($config_file and -e $config_file) { 549 print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . 550 " because config file ($config_file) is present\n"; 551 $status = 0; 552 } 496 553 } 497 554 if ($status) { … … 529 586 if ($status) { 530 587 my $command = "$difftool -diff_id $stage_id"; 531 # my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; 588 532 589 if ($mode eq "goto_purged") { 533 590 $command .= " -updaterun -state purged"; 534 } else { 591 } 592 elsif ($mode eq "goto_cleaned") { 535 593 $command .= " -updaterun -state cleaned"; 536 594 } 595 elsif ($mode eq "goto_scrubbed") { 596 $command .= " -updaterun -state scrubbed"; 597 } 598 537 599 $command .= " -dbname $dbname" if defined $dbname; 538 600 … … 545 607 } else { 546 608 my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state"; 547 # my $command = "$difftool -updaterun -diff_id $stage_id -skycell_id $skycell_id -state $error_state"; 609 548 610 $command .= " -dbname $dbname" if defined $dbname; 549 611 … … 560 622 } 561 623 if ($stage eq 'fake') { 562 die "ipp_cleanup.pl -stage fake not yet implemented. Probably will just mark database cleaned.\n"; 624 print STDERR "This does not seem to work at present, as no files exist. Terminating quietly.\n"; 625 exit(0); 626 die "--stage_id required for stage fake\n" if !$stage_id; 627 ### select the imfiles for this entry 628 629 # this stage uses 'chiptool' 630 my $faketool = can_run('faketool') or die "Can't find faketool"; 631 632 # Get list of component imfiles 633 # XXX may need a different my_die for each stage 634 my $imfiles; # Array of component files 635 my $command = "$faketool -pendingcleanupimfile -fake_id $stage_id"; # Command to run 636 $command .= " -dbname $dbname" if defined $dbname; 637 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 638 unless ($success) { 639 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 640 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 641 } 642 643 # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' 644 if (@$stdout_buf == 0) { 645 my $command = "$faketool -fake_id $stage_id -updaterun -set_state new"; 646 $command .= " -dbname $dbname" if defined $dbname; 647 648 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 649 run(command => $command, verbose => $verbose); 650 unless ($success) { 651 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 652 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 653 } 654 exit 0; 655 } 656 657 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 658 &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR); 659 660 # extract the metadata for the files into a hash list 661 $imfiles = parse_md_list($metadata) or 662 &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR); 663 664 # loop over all of the imfiles, determine the path_base and class_id for each 665 foreach my $imfile (@$imfiles) { 666 my $class_id = $imfile->{class_id}; 667 my $path_base = $imfile->{path_base}; 668 my $status = 1; 669 670 # don't clean up unless the data needed to update is available 671 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 672 # goto_scrubbed now requires the config file to not exist. 673 if ($mode eq "goto_cleaned") { 674 my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); 675 676 if (!$config_file or ! -e $config_file) { 677 print STDERR "skipping cleanup for fakeRun $stage_id $class_id " 678 . " because config file is missing\n"; 679 $status = 0; 680 } 681 } 682 elsif ($mode eq "goto_scrubbed") { 683 my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); 684 685 if ($config_file and -e $config_file) { 686 print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " 687 . " because config file is present\n"; 688 $status = 0; 689 } 690 } 691 692 if ($status) { 693 # array of actual filenames to delete 694 my @files = (); 695 696 # delete the temporary image datafiles 697 addFilename (\@files, "PPSIM.OUTPUT.MEF", $path_base, $class_id); 698 addFilename (\@files, "PPSIM.OUTPUT.SPL", $path_base, $class_id); 699 addFilename (\@files, "PPSIM.FAKE.CHIP", $path_base, $class_id); 700 addFilename (\@files, "PPSIM.FORCE.CHIP", $path_base, $class_id); 701 if ($mode eq "goto_purged") { 702 # additional files to remove for 'purge' mode 703 addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id); 704 addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id); 705 addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id); 706 } 707 708 # actual command to delete the files 709 $status = &delete_files (\@files); 710 } 711 712 if ($status) { 713 my $command = "$faketool -fake_id $stage_id -class_id $class_id"; 714 if ($mode eq "goto_purged") { 715 $command .= " -topurgedimfile"; 716 } 717 elsif ($mode eq "goto_cleaned") { 718 $command .= " -tocleanedimfile"; 719 } 720 elsif ($mode eq "goto_scrubbed") { 721 $command .= " -toscrubbedimfile"; 722 } 723 724 $command .= " -dbname $dbname" if defined $dbname; 725 726 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 727 run(command => $command, verbose => $verbose); 728 unless ($success) { 729 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 730 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 731 } 732 } else { 733 734 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) 735 my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state"; 736 $command .= " -dbname $dbname" if defined $dbname; 737 738 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 739 run(command => $command, verbose => $verbose); 740 unless ($success) { 741 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 742 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 743 } 744 } 745 } 746 exit 0; 747 563 748 } 564 749 # fake : faketool : -pendingcleanupimfile (loop over imfiles)
Note:
See TracChangeset
for help on using the changeset viewer.
