Changeset 29671 for trunk/ippScripts/scripts/ipp_cleanup.pl
- Timestamp:
- Nov 4, 2010, 1:42:23 PM (16 years ago)
- File:
-
- 1 edited
-
trunk/ippScripts/scripts/ipp_cleanup.pl (modified) (5 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/ipp_cleanup.pl
r29420 r29671 18 18 19 19 # Parse the command-line arguments 20 my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist );20 my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile); 21 21 GetOptions('stage=s' => \$stage, # which analysis stage to clean? 22 22 'camera|i=s' => \$camera, # user-supplied camera name … … 27 27 'verbose' => \$verbose, # Print to stdout 28 28 'no-op' => \$no_op, # pretend but don't actually inject 29 'helplist' => \$helplist # give help listing 29 'helplist' => \$helplist, # give help listing 30 'logfile=s' => \$logfile # destination for stdout and stderr 30 31 ) or pod2usage( 2 ); 31 32 … … 53 54 } 54 55 56 $ipprc->redirect_output($logfile) or 57 &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile; 58 55 59 # set this to 1 to enable checking for files on dead nodes 56 60 # it is off for now because the implementation is a hack 57 61 # See comments below. 58 my $check_for_gone = 0;62 my $check_for_gone = 1; 59 63 60 64 my $error_state; … … 66 70 67 71 my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, 72 "chip_bg" => 1, "warp_bg" => 1, 68 73 "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0, 69 74 "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 ); … … 1648 1653 } 1649 1654 1655 if ($stage eq "chip_bg") { 1656 1657 die "--stage_id required for stage chip_bg\n" if !$stage_id; 1658 1659 &my_die("only mode goto_cleaned is supported for stage chip_bg", "chip_bg", $stage_id, $PS_EXIT_PROG_ERROR) 1660 if $mode ne 'goto_cleaned'; 1661 1662 ### select the imfiles for this entry 1663 1664 # this stage uses 'bgtool' 1665 my $bgtool = can_run('bgtool') or die "Can't find bgtool"; 1666 1667 # Get list of component imfiles 1668 # XXX may need a different my_die for each stage 1669 my $imfiles; # Array of component files 1670 my $command = "$bgtool -pendingcleanupchipimfile -chip_bg_id $stage_id"; # Command to run 1671 $command .= " -dbname $dbname" if defined $dbname; 1672 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1673 unless ($success) { 1674 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1675 &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code); 1676 } 1677 1678 # XXX: Is not having any files to process really a bug? 1679 if (@$stdout_buf == 0) { 1680 my $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state"; 1681 $command .= " -dbname $dbname" if defined $dbname; 1682 1683 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1684 run(command => $command, verbose => $verbose); 1685 unless ($success) { 1686 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1687 &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code); 1688 } 1689 exit 0; 1690 } 1691 1692 # extract the metadata for the files into a hash list 1693 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 1694 &my_die("Unable to parse metadata config doc", "chip_bg", $stage_id, $PS_EXIT_PROG_ERROR); 1695 1696 # loop over all of the imfiles, determine the path_base and class_id for each 1697 foreach my $imfile (@$imfiles) { 1698 my $class_id = $imfile->{class_id}; 1699 my $path_base = $imfile->{path_base}; 1700 my $status = 1; 1701 $status = 0 unless defined $path_base and $path_base ne "NULL"; 1702 1703 # don't clean up unless the data needed to update is available 1704 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 1705 # goto_scrubbed now requires the config file to not exist. 1706 if ($status) { 1707 if ($mode eq "goto_cleaned") { 1708 my $config_file = $ipprc->filename("PPBACKGROUND.CONFIG", $path_base, $class_id); 1709 1710 unless ($ipprc->file_exists($config_file)) { 1711 if (file_gone($config_file)) { 1712 print STDERR "forcing cleanup for chipRun $stage_id $class_id " 1713 . " because config file ($config_file) is gone\n"; 1714 } else { 1715 print STDERR "skipping cleanup for chipRun $stage_id $class_id " 1716 . " because config file ($config_file) is missing\n"; 1717 $status = 0; 1718 } 1719 } 1720 } 1721 elsif ($mode eq "goto_scrubbed") { 1722 my $config_file = $ipprc->filename("PPBACKGROUND.CONFIG", $path_base, $class_id); 1723 1724 if ($ipprc->file_exists($config_file)) { 1725 print STDERR "skipping scrubbed for chipBackgroundRun $stage_id $class_id " 1726 . " because config file ($config_file) is present\n"; 1727 $status = 0; 1728 } 1729 } 1730 } 1731 1732 if ($status) { 1733 # array of actual filenames to delete 1734 my @files = (); 1735 1736 # delete the image datafiles 1737 addFilename (\@files, "PPBACKGROUND.OUTPUT", $path_base, $class_id); 1738 addFilename (\@files, "PPBACKGROUND.OUTPUT.MASK", $path_base, $class_id); 1739 if ($mode eq "goto_purged") { 1740 # additional files to remove for 'purge' mode 1741 addFilename (\@files, "PPBACKGROUND.STATS", $path_base, $class_id); #clean? 1742 addFilename (\@files, "PPBACKGOROUND.CONFIG", $path_base, $class_id); 1743 } 1744 1745 # actual command to delete the files 1746 $status = &delete_files (\@files); 1747 } 1748 1749 if ($status) { 1750 my $command = "$bgtool -chip_bg_id $stage_id -class_id $class_id"; 1751 if ($mode eq "goto_cleaned") { 1752 $command .= " -tocleanedchipimfile"; 1753 } 1754 elsif ($mode eq "goto_purged") { 1755 $command .= " -topurgedchipimfile"; 1756 } 1757 elsif ($mode eq "goto_scrubbed") { 1758 $command .= " -toscrubbedchipimfile"; 1759 } 1760 1761 $command .= " -dbname $dbname" if defined $dbname; 1762 1763 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1764 run(command => $command, verbose => $verbose); 1765 unless ($success) { 1766 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1767 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 1768 } 1769 1770 } else { 1771 1772 # if an error happens for one chip, the chipBackgroundRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) 1773 my $command = "$bgtool -updatechipimfile -chip_bg_id $stage_id -class_id $class_id -set_data_state $error_state"; 1774 $command .= " -dbname $dbname" if defined $dbname; 1775 1776 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1777 run(command => $command, verbose => $verbose); 1778 unless ($success) { 1779 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1780 &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code); 1781 } 1782 1783 # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. 1784 $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state"; 1785 $command .= " -dbname $dbname" if defined $dbname; 1786 1787 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1788 run(command => $command, verbose => $verbose); 1789 unless ($success) { 1790 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1791 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 1792 } 1793 1794 } 1795 } 1796 exit 0; 1797 } 1798 if ($stage eq "warp_bg") { 1799 1800 die "--stage_id required for stage warp_bg\n" if !$stage_id; 1801 1802 &my_die("only mode goto_cleaned is supported for stage warp_bg", "warp_bg", $stage_id, $PS_EXIT_PROG_ERROR) 1803 if $mode ne 'goto_cleaned'; 1804 1805 ### select the imfiles for this entry 1806 1807 # this stage uses 'bgtool' 1808 my $bgtool = can_run('bgtool') or die "Can't find bgtool"; 1809 1810 # Get list of component imfiles 1811 # XXX may need a different my_die for each stage 1812 my $imfiles; # Array of component files 1813 my $command = "$bgtool -pendingcleanupwarpskyfile -warp_bg_id $stage_id"; # Command to run 1814 $command .= " -dbname $dbname" if defined $dbname; 1815 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1816 unless ($success) { 1817 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1818 &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code); 1819 } 1820 1821 # XXX: Is not having any files to process really a bug? 1822 if (@$stdout_buf == 0) { 1823 my $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state"; 1824 $command .= " -dbname $dbname" if defined $dbname; 1825 1826 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1827 run(command => $command, verbose => $verbose); 1828 unless ($success) { 1829 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1830 &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code); 1831 } 1832 exit 0; 1833 } 1834 1835 # extract the metadata for the files into a hash list 1836 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 1837 &my_die("Unable to parse metadata config doc", "warp_bg", $stage_id, $PS_EXIT_PROG_ERROR); 1838 1839 # loop over all of the imfiles, determine the path_base and skycell_id for each 1840 foreach my $imfile (@$imfiles) { 1841 my $skycell_id = $imfile->{skycell_id}; 1842 my $path_base = $imfile->{path_base}; 1843 my $status = 1; 1844 $status = 0 unless defined $path_base and $path_base ne "NULL"; 1845 1846 # don't clean up unless the data needed to update is available 1847 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 1848 # goto_scrubbed now requires the config file to not exist. 1849 if ($status) { 1850 if ($mode eq "goto_cleaned") { 1851 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 1852 1853 unless ($ipprc->file_exists($config_file)) { 1854 if (file_gone($config_file)) { 1855 print STDERR "forcing cleanup for warpBackgroundSkyfile $stage_id $skycell_id " 1856 . " because config file ($config_file) is gone\n"; 1857 } else { 1858 print STDERR "skipping cleanup for warpBackgroundSkyfile $stage_id $skycell_id " 1859 . " because config file ($config_file) is missing\n"; 1860 $status = 0; 1861 } 1862 } 1863 } 1864 elsif ($mode eq "goto_scrubbed") { 1865 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 1866 1867 if ($ipprc->file_exists($config_file)) { 1868 print STDERR "skipping scrubbed for chipBackgroundRun $stage_id $skycell_id " 1869 . " because config file ($config_file) is present\n"; 1870 $status = 0; 1871 } 1872 } 1873 } 1874 1875 if ($status) { 1876 # array of actual filenames to delete 1877 my @files = (); 1878 1879 # delete the image datafiles 1880 addFilename (\@files, "PSWARP.OUTPUT", $path_base, $skycell_id); 1881 addFilename (\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); 1882 if ($mode eq "goto_purged") { 1883 # additional files to remove for 'purge' mode 1884 addFilename (\@files, "PSWARP.STATS", $path_base, $skycell_id); #clean? 1885 addFilename (\@files, "PSWARP.CONFIG", $path_base, $skycell_id); 1886 } 1887 1888 # actual command to delete the files 1889 $status = &delete_files (\@files); 1890 } 1891 1892 if ($status) { 1893 my $command = "$bgtool -warp_bg_id $stage_id -skycell_id $skycell_id"; 1894 # recall that only goto_cleaned is supported (currently) 1895 if ($mode eq "goto_cleaned") { 1896 $command .= " -tocleanedwarpskyfile"; 1897 } 1898 elsif ($mode eq "goto_purged") { 1899 $command .= " -topurgedwarpskyfile"; 1900 } 1901 elsif ($mode eq "goto_scrubbed") { 1902 $command .= " -toscrubbedwarpskyfile"; 1903 } 1904 1905 $command .= " -dbname $dbname" if defined $dbname; 1906 1907 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1908 run(command => $command, verbose => $verbose); 1909 unless ($success) { 1910 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1911 &my_die("Unable to perform bgtool: $error_code", "warp", $stage_id, $error_code); 1912 } 1913 1914 } else { 1915 1916 # if an error happens for one skycell, the warpBackgroundRun will stay in goto_*, but the skycells will go to error_* (matching the goto_*) 1917 my $command = "$bgtool -updatewarpskyfile -warp_bg_id $stage_id -skycell_id $skycell_id -set_data_state $error_state"; 1918 $command .= " -dbname $dbname" if defined $dbname; 1919 1920 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1921 run(command => $command, verbose => $verbose); 1922 unless ($success) { 1923 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1924 &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code); 1925 } 1926 1927 # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. 1928 $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state"; 1929 $command .= " -dbname $dbname" if defined $dbname; 1930 1931 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1932 run(command => $command, verbose => $verbose); 1933 unless ($success) { 1934 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1935 &my_die("Unable to perform bgtool: $error_code", "warp", $stage_id, $error_code); 1936 } 1937 } 1938 } 1939 exit 0; 1940 } 1650 1941 1651 1942 die "ipp_cleanup.pl -stage $stage not yet implemented\n";
Note:
See TracChangeset
for help on using the changeset viewer.
