- Timestamp:
- Jul 13, 2009, 6:24:00 PM (17 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/czw_branch/cleanup/ippScripts/scripts/ipp_cleanup.pl
r24743 r24785 747 747 748 748 } 749 # fake : faketool : -pendingcleanupimfile (loop over imfiles) 749 # Detrend stages 750 if ($stage eq "detrend.process.imfile") { 751 752 die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id; 753 ### select the imfiles for this entry 754 755 # this stage uses 'dettool' 756 my $dettool = can_run('dettool') or die "Can't find chiptool"; 757 758 # Get list of component imfiles 759 # XXX may need a different my_die for each stage 760 my $imfiles; # Array of component files 761 my $command = "$dettool -pendingcleanup_processedimfile -chip_id $stage_id"; # Command to run 762 $command .= " -dbname $dbname" if defined $dbname; 763 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 764 unless ($success) { 765 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 766 &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); 767 } 768 769 # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' 770 if (@$stdout_buf == 0) { 771 my $command = "$dettool -det_id $stage_id -updateprocessedimfile -data_state new"; 772 $command .= " -dbname $dbname" if defined $dbname; 773 774 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 775 run(command => $command, verbose => $verbose); 776 unless ($success) { 777 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 778 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 779 } 780 exit 0; 781 } 782 783 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 784 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 785 786 # extract the metadata for the files into a hash list 787 $imfiles = parse_md_list($metadata) or 788 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 789 790 # loop over all of the imfiles, determine the path_base and class_id for each 791 foreach my $imfile (@$imfiles) { 792 my $exp_id = $imfile->{exp_id}; 793 my $class_id = $imfile->{class_id}; 794 my $path_base = $imfile->{path_base}; 795 my $status = 1; 796 797 # don't clean up unless the data needed to update is available 798 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 799 # goto_scrubbed now requires the config file to not exist. 800 801 # Possibly not the correct config file, but simtest doesn't leave any around to check. 802 if ($mode eq "goto_cleaned") { 803 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 804 805 if (!$config_file or ! -e $config_file) { 806 print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id " 807 . " because config file is missing\n"; 808 $status = 0; 809 } 810 } 811 elsif ($mode eq "goto_scrubbed") { 812 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 813 814 if ($config_file and -e $config_file) { 815 print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id " 816 . " because config file is present\n"; 817 $status = 0; 818 } 819 } 820 821 if ($status) { 822 # array of actual filenames to delete 823 my @files = (); 824 825 # delete the temporary image datafiles 826 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); 827 addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); 828 addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); 829 addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id); 830 addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id); 831 addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id); 832 if ($mode eq "goto_purged") { 833 # additional files to remove for 'purge' mode 834 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); 835 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); 836 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 837 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 838 addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); 839 addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id); 840 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 841 addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); 842 } 843 844 # actual command to delete the files 845 $status = &delete_files (\@files); 846 } 847 848 if ($status) { 849 my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile"; 850 if ($mode eq "goto_purged") { 851 $command .= " -data_state purged"; 852 } 853 elsif ($mode eq "goto_cleaned") { 854 $command .= " -data_state cleaned"; 855 } 856 elsif ($mode eq "goto_scrubbed") { 857 $command .= " -data_state scrubbed"; 858 } 859 860 $command .= " -dbname $dbname" if defined $dbname; 861 862 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 863 run(command => $command, verbose => $verbose); 864 unless ($success) { 865 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 866 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 867 } 868 } else { 869 870 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) 871 my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state"; 872 $command .= " -dbname $dbname" if defined $dbname; 873 874 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 875 run(command => $command, verbose => $verbose); 876 unless ($success) { 877 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 878 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 879 } 880 } 881 } 882 exit 0; 883 } 884 if ($stage eq "detrend.process.exp") { 885 die "--stage_id required for stage $stage\n" if !$stage_id; 886 # this stage uses 'camtool' 887 my $camtool = can_run('dettool') or die "Can't find dettool"; 888 889 # Get list of component imfiles 890 # XXX may need a different my_die for each stage 891 my $exps; # Array of component files 892 my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run 893 $command .= " -dbname $dbname" if defined $dbname; 894 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 895 unless ($success) { 896 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 897 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 898 } 899 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 900 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 901 902 $exps = parse_md_list($metadata) or 903 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 904 905 906 foreach my $exp (@$exps) { 907 my $path_base = $exp->{path_base}; 908 my $exp_id = $exp->{exp_id}; 909 910 my $status = 1; 911 # don't clean up unless the data needed to update is available 912 # goto_scrubbed now requires the config file to not be present 913 if ($mode eq "goto_cleaned") { 914 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); 915 916 if (!$config_file or ! -e $config_file) { 917 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 918 $status = 0; 919 } 920 } 921 elsif ($mode eq "goto_scrubbed") { 922 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); 923 924 if ($config_file and -e $config_file) { 925 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 926 $status = 0; 927 } 928 } 929 if ($status) { 930 my @files = (); 931 # delete the temporary image datafiles 932 addFilename (\@files, "PSASTRO.OUTPUT", $path_base); 933 if ($mode eq "goto_purged") { 934 # additional files to remove for 'purge' mode 935 addFilename (\@files, "PPIMAGE.JPEG1", $path_base); 936 addFilename (\@files, "PPIMAGE.JPEG2", $path_base); 937 addFilename (\@files, "PSASTRO.STATS", $path_base); 938 } 939 # actual command to delete the files 940 $status = &delete_files (\@files); 941 } 942 943 if ($status) { 944 my $command; 945 if ($mode eq "goto_cleaned") { 946 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned"; 947 } 948 if ($mode eq "goto_scrubbed") { 949 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed"; 950 } 951 if ($mode eq "goto_purged") { 952 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged"; 953 } 954 $command .= " -dbname $dbname" if defined $dbname; 955 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 956 run(command => $command, verbose => $verbose); 957 unless ($success) { 958 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 959 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 960 } 961 } else { 962 # since 'camera' has only a single imfile, we can just update the run 963 my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state"; 964 $command .= " -dbname $dbname" if defined $dbname; 965 966 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 967 run(command => $command, verbose => $verbose); 968 unless ($success) { 969 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 970 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 971 } 972 exit $PS_EXIT_UNKNOWN_ERROR; 973 } 974 } 975 exit 0; 976 } 977 if ($stage eq "detrend.stack.imfile") { 978 979 die "--stage_id required for stage $stage\n" if !$stage_id; 980 981 # this stage uses 'stacktool' 982 my $stacktool = can_run('dettool') or die "Can't find dettool"; 983 984 # Get list of component imfiles 985 my $skyfiles; # Array reference of component files 986 my $command = "dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run 987 $command .= " -dbname $dbname" if defined $dbname; 988 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 989 unless ($success) { 990 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 991 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 992 } 993 my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 994 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 995 996 $skyfiles = parse_md_list($metadata) or 997 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 998 999 my @files = (); 1000 foreach my $stack (@{ $stacks }) { 1001 # detStackedImfile does not have a path_base column. This is inconvenient, as it means we need to calculate it. 1002 my $path_base = $stack->{uri}; 1003 my $iteration = $stack->{iteration}; 1004 my $class_id = $stack->{class_id}; 1005 1006 $path_base =~ s/\.fits$//; # That should do it? 1007 1008 my $status = 1; 1009 if ($mode eq "goto_cleaned") { 1010 my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); 1011 1012 $config_file =~ s%^file://%%; 1013 if (!$config_file or ! -e $config_file) { 1014 print STDERR "skipping cleanup for $stage $stage_id $path_base" . 1015 " because config file is missing\n"; 1016 $status = 0; 1017 } 1018 $config_file = 'file://' . $config_file; 1019 } 1020 elsif ($mode eq "goto_scrubbed") { 1021 my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); 1022 $config_file =~ s%^file://%%; 1023 if ($config_file and -e $config_file) { 1024 print STDERR "skipping scrubbed for $stage $stage_id $path_base" . 1025 " because config file is present\n"; 1026 $status = 0; 1027 } 1028 } 1029 if ($status) { 1030 # delete the temporary image datafiles 1031 addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $skycell_id); 1032 addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $skycell_id); 1033 addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $skycell_id); 1034 1035 if ($mode eq "goto_purged") { 1036 # additional files to remove for 'purge' mode 1037 # Nothing? 1038 } 1039 1040 $status = &delete_files(\@files); 1041 } 1042 1043 if ($status) { 1044 my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id"; 1045 if ($mode eq "goto_purged") { 1046 $command .= " -updatestacked -data_state purged"; 1047 } 1048 elsif ($mode eq "goto_cleaned") { 1049 $command .= " -updatestacked -data_state cleaned"; 1050 } 1051 elsif ($mode eq "goto_scrubbed") { 1052 $command .= " -updatestacked -data_state scrubbed"; 1053 } 1054 $command .= " -dbname $dbname" if defined $dbname; 1055 1056 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1057 run(command => $command, verbose => $verbose); 1058 unless ($success) { 1059 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1060 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1061 } 1062 } else { 1063 my $command = "$dettool -updatestacked -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1064 $command .= " -dbname $dbname" if defined $dbname; 1065 1066 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1067 run(command => $command, verbose => $verbose); 1068 unless ($success) { 1069 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1070 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1071 } 1072 exit $PS_EXIT_UNKNOWN_ERROR; 1073 } 1074 } 1075 exit 0; 1076 } 1077 if ($stage eq "detrend.normstat.imfile") { 1078 print STDERR "I'm not convinced there's anything to clean up from stage $stage\n"; 1079 die "--stage_id required for stage $stage\n" if !$stage_id; 1080 # this stage uses 'camtool' 1081 my $camtool = can_run('dettool') or die "Can't find dettool"; 1082 1083 my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run 1084 $command .= " -dbname $dbname" if defined $dbname; 1085 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1086 unless ($success) { 1087 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1088 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1089 } 1090 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1091 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1092 1093 $exps = parse_md_list($metadata) or 1094 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1095 1096 foreach my $exp (@$exps) { 1097 my $path_base = $exp->{path_base}; 1098 my $iteration = $exp->{iteration}; 1099 my $class_id = $exp->{class_id}; 1100 1101 my $status = 1; 1102 if ($status) { 1103 my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id"; 1104 if ($mode eq "goto_cleaned") { 1105 $command .= " -data_state cleaned"; 1106 } 1107 if ($mode eq "goto_scrubbed") { 1108 $command .= " -data_state scrubbed"; 1109 } 1110 if ($mode eq "goto_purged") { 1111 $command .= " -data_state purged"; 1112 } 1113 $command .= " -dbname $dbname" if defined $dbname; 1114 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1115 run(command => $command, verbose => $verbose); 1116 unless ($success) { 1117 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1118 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1119 } 1120 } else { 1121 # since 'camera' has only a single imfile, we can just update the run 1122 my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1123 $command .= " -dbname $dbname" if defined $dbname; 1124 1125 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1126 run(command => $command, verbose => $verbose); 1127 unless ($success) { 1128 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1129 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1130 } 1131 exit $PS_EXIT_UNKNOWN_ERROR; 1132 } 1133 } 1134 exit 0; 1135 1136 } 1137 if ($stage eq "detrend.norm.imfile") { 1138 die "--stage_id required for stage $stage\n" if !$stage_id; 1139 # this stage uses 'camtool' 1140 my $camtool = can_run('dettool') or die "Can't find dettool"; 1141 1142 # Get list of component imfiles 1143 # XXX may need a different my_die for each stage 1144 my $exps; # Array of component files 1145 my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run 1146 $command .= " -dbname $dbname" if defined $dbname; 1147 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1148 unless ($success) { 1149 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1150 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1151 } 1152 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1153 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1154 1155 $exps = parse_md_list($metadata) or 1156 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1157 1158 foreach my $exp (@$exps) { 1159 my $path_base = $exp->{path_base}; 1160 my $iteration = $exp->{iteration}; 1161 my $class_id = $exp->{class_id}; 1162 1163 my $status = 1; 1164 # don't clean up unless the data needed to update is available 1165 # goto_scrubbed now requires the config file to not be present 1166 if ($mode eq "goto_cleaned") { 1167 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1168 1169 if (!$config_file or ! -e $config_file) { 1170 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 1171 $status = 0; 1172 } 1173 } 1174 elsif ($mode eq "goto_scrubbed") { 1175 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1176 1177 if ($config_file and -e $config_file) { 1178 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 1179 $status = 0; 1180 } 1181 } 1182 if ($status) { 1183 my @files = (); 1184 # delete the temporary image datafiles 1185 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base); 1186 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base); 1187 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base); 1188 if ($mode eq "goto_purged") { 1189 # additional files to remove for 'purge' mode 1190 addFilename (\@files, "PPIMAGE.JPEG1", $path_base); 1191 addFilename (\@files, "PPIMAGE.JPEG2", $path_base); 1192 } 1193 # actual command to delete the files 1194 $status = &delete_files (\@files); 1195 } 1196 1197 if ($status) { 1198 my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id"; 1199 if ($mode eq "goto_cleaned") { 1200 $command .= " -data_state cleaned"; 1201 } 1202 if ($mode eq "goto_scrubbed") { 1203 $command .= " -data_state scrubbed"; 1204 } 1205 if ($mode eq "goto_purged") { 1206 $command .= " -data_state purged"; 1207 } 1208 $command .= " -dbname $dbname" if defined $dbname; 1209 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1210 run(command => $command, verbose => $verbose); 1211 unless ($success) { 1212 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1213 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1214 } 1215 } else { 1216 my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1217 $command .= " -dbname $dbname" if defined $dbname; 1218 1219 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1220 run(command => $command, verbose => $verbose); 1221 unless ($success) { 1222 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1223 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1224 } 1225 exit $PS_EXIT_UNKNOWN_ERROR; 1226 } 1227 } 1228 exit 0; 1229 } 1230 if ($stage eq "detrend.resid.imfile") { 1231 1232 die "--stage_id required for stage $stage\n" if !$stage_id; 1233 ### select the imfiles for this entry 1234 1235 # this stage uses 'dettool' 1236 my $dettool = can_run('dettool') or die "Can't find dettool"; 1237 1238 # Get list of component imfiles 1239 # XXX may need a different my_die for each stage 1240 my $imfiles; # Array of component files 1241 my $command = "$dettool -pendingcleanup_residimfile -chip_id $stage_id"; # Command to run 1242 $command .= " -dbname $dbname" if defined $dbname; 1243 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1244 unless ($success) { 1245 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1246 &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); 1247 } 1248 1249 # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' 1250 if (@$stdout_buf == 0) { 1251 my $command = "$dettool -det_id $stage_id -updateresidimfile -data_state new"; 1252 $command .= " -dbname $dbname" if defined $dbname; 1253 1254 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1255 run(command => $command, verbose => $verbose); 1256 unless ($success) { 1257 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1258 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1259 } 1260 exit 0; 1261 } 1262 1263 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1264 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1265 1266 # extract the metadata for the files into a hash list 1267 $imfiles = parse_md_list($metadata) or 1268 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1269 1270 # loop over all of the imfiles, determine the path_base and class_id for each 1271 foreach my $imfile (@$imfiles) { 1272 my $class_id = $imfile->{class_id}; 1273 my $iteration = $imfile->{iteration}; 1274 my $exp_id = $imfile->{exp_id}; 1275 my $path_base = $imfile->{path_base}; 1276 my $status = 1; 1277 1278 # don't clean up unless the data needed to update is available 1279 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 1280 # goto_scrubbed now requires the config file to not exist. 1281 1282 # Possibly not the correct config file, but simtest doesn't leave any around to check. 1283 if ($mode eq "goto_cleaned") { 1284 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 1285 1286 if (!$config_file or ! -e $config_file) { 1287 print STDERR "skipping cleanup for $stage $stage_id $class_id " 1288 . " because config file is missing\n"; 1289 $status = 0; 1290 } 1291 } 1292 elsif ($mode eq "goto_scrubbed") { 1293 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 1294 1295 if ($config_file and -e $config_file) { 1296 print STDERR "skipping scrubbed for $stage $stage_id $class_id " 1297 . " because config file is present\n"; 1298 $status = 0; 1299 } 1300 } 1301 1302 if ($status) { 1303 # array of actual filenames to delete 1304 my @files = (); 1305 1306 # delete the temporary image datafiles 1307 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); 1308 addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id); 1309 addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id); 1310 addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id); 1311 if ($mode eq "goto_purged") { 1312 # additional files to remove for 'purge' mode 1313 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); 1314 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); 1315 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 1316 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 1317 addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); 1318 addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id); 1319 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 1320 addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); 1321 } 1322 1323 # actual command to delete the files 1324 $status = &delete_files (\@files); 1325 } 1326 1327 if ($status) { 1328 my $command = "$dettool -updateresidimfile -chip_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id"; 1329 if ($mode eq "goto_purged") { 1330 $command .= " -data_state purged"; 1331 } 1332 elsif ($mode eq "goto_cleaned") { 1333 $command .= " -data_state cleaned"; 1334 } 1335 elsif ($mode eq "goto_scrubbed") { 1336 $command .= " -data_state scrubbed"; 1337 } 1338 1339 $command .= " -dbname $dbname" if defined $dbname; 1340 1341 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1342 run(command => $command, verbose => $verbose); 1343 unless ($success) { 1344 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1345 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1346 } 1347 } else { 1348 my $command = "$dettool -updateresidimfile -chip_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1349 $command .= " -dbname $dbname" if defined $dbname; 1350 1351 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1352 run(command => $command, verbose => $verbose); 1353 unless ($success) { 1354 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1355 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1356 } 1357 } 1358 } 1359 exit 0; 1360 } 1361 if ($stage eq "detrend.resid.exp") { 1362 die "--stage_id required for stage $stage\n" if !$stage_id; 1363 # this stage uses 'camtool' 1364 my $camtool = can_run('dettool') or die "Can't find dettool"; 1365 1366 # Get list of component imfiles 1367 # XXX may need a different my_die for each stage 1368 my $exps; # Array of component files 1369 my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run 1370 $command .= " -dbname $dbname" if defined $dbname; 1371 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1372 unless ($success) { 1373 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1374 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1375 } 1376 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1377 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1378 1379 $exps = parse_md_list($metadata) or 1380 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1381 1382 foreach my $exp (@$exps) { 1383 my $exp_id = $exp->{exp_id}; 1384 my $iteration = $exp->{iteration}; 1385 my $path_base = $exp->{path_base}; 1386 1387 my $status = 1; 1388 # don't clean up unless the data needed to update is available 1389 # goto_scrubbed now requires the config file to not be present 1390 if ($mode eq "goto_cleaned") { 1391 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); 1392 1393 if (!$config_file or ! -e $config_file) { 1394 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 1395 $status = 0; 1396 } 1397 } 1398 elsif ($mode eq "goto_scrubbed") { 1399 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); 1400 1401 if ($config_file and -e $config_file) { 1402 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 1403 $status = 0; 1404 } 1405 } 1406 if ($status) { 1407 my @files = (); 1408 # delete the temporary image datafiles 1409 addFilename (\@files, "PSASTRO.OUTPUT", $path_base); 1410 if ($mode eq "goto_purged") { 1411 # additional files to remove for 'purge' mode 1412 addFilename (\@files, "PPIMAGE.JPEG1", $path_base); 1413 addFilename (\@files, "PPIMAGE.JPEG2", $path_base); 1414 addFilename (\@files, "PSASTRO.STATS", $path_base); 1415 } 1416 # actual command to delete the files 1417 $status = &delete_files (\@files); 1418 } 1419 1420 if ($status) { 1421 my $command = "$dettool -updateresiddexp -det_id $stage_id -exp_id $exp_id -iteration $iteration"; 1422 if ($mode eq "goto_cleaned") { 1423 $command .= " -data_state cleaned"; 1424 } 1425 if ($mode eq "goto_scrubbed") { 1426 $command .= " -data_state scrubbed"; 1427 } 1428 if ($mode eq "goto_purged") { 1429 $command .= " -data_state purged"; 1430 } 1431 $command .= " -dbname $dbname" if defined $dbname; 1432 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1433 run(command => $command, verbose => $verbose); 1434 unless ($success) { 1435 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1436 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1437 } 1438 } else { 1439 my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; 1440 $command .= " -dbname $dbname" if defined $dbname; 1441 1442 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1443 run(command => $command, verbose => $verbose); 1444 unless ($success) { 1445 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1446 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1447 } 1448 exit $PS_EXIT_UNKNOWN_ERROR; 1449 } 1450 } 1451 exit 0; 1452 } 1453 750 1454 751 1455 die "ipp_cleanup.pl -stage $stage not yet implemented\n";
Note:
See TracChangeset
for help on using the changeset viewer.
