Changeset 9081
- Timestamp:
- Sep 30, 2006, 12:27:01 PM (20 years ago)
- Location:
- trunk/ippTasks
- Files:
-
- 1 added
- 5 edited
-
detrend.process.pro (modified) (5 diffs)
-
detrend.reject.pro (modified) (6 diffs)
-
detrend.resid.pro (modified) (6 diffs)
-
detrend.stack.pro (modified) (6 diffs)
-
pantasks.pro (added)
-
phase0.pro (modified) (9 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippTasks/detrend.process.pro
r8691 r9081 13 13 ## STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI 14 14 15 queueinit DetrendImfilesToProcess 16 if ($?network == 0) 17 $network = 1 18 end 19 if ($?parallel == 0) 20 $parallel = 0 21 end 22 15 23 # select images ready for copy 16 24 # new entries are added to queue DetrendImfilesToProcess … … 23 31 periods -exec 5 24 32 periods -timeout 30 25 trange 18:00 23:59 33 34 stdout dettool.process.log 35 stderr dettool.process.log 26 36 27 37 # success … … 69 79 task dettool.raw.process 70 80 periods -poll 0.5 71 periods -exec 1.072 periods -timeout 581 periods -exec 2.0 82 periods -timeout 60 73 83 74 84 task.exec … … 90 100 ## NOTE: the queue has the STATE prepended 91 101 list word -split $line 92 $DET_ID = $word:1 93 $TYPE = $word:2 94 $EXP_KEY = $word:3 95 $CLASS = $word:4 96 $CLASS_ID = $word:5 97 $URI = $word:6 102 $DET_ID = $word:0 103 $TYPE = $word:1 104 $EXP_KEY = $word:2 105 $CLASS = $word:3 106 $CLASS_ID = $word:4 107 $URI = $word:5 108 109 stdout $EXP_KEY.detproc.$CLASS_ID.log 110 stderr $EXP_KEY.detproc.$CLASS_ID.log 98 111 99 112 # specify choice of remote host: 100 host anyhost 113 if ($parallel) 114 host anyhost 115 else 116 host local 117 end 101 118 102 119 # create example job options as a demonstration 103 120 options "$line" 104 # ~/ipp/helpers/detrend_process.pl 1 850131b mc bias 850131b.fits 850131b105 command detrend_process.pl $DET_ID $EXP_KEY $CLASS_ID $TYPE $URI $EXP_KEY121 echo command detrend_process.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI 122 command detrend_process.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI 106 123 end 107 124 … … 118 135 # operation times out? 119 136 task.exit timeout 120 queuepush DetrendImfilesToProcess -replace -key 1:3:5 " FAIL$options:0"137 queuepush DetrendImfilesToProcess -replace -key 1:3:5 "TIMEOUT $options:0" 121 138 end 122 139 end -
trunk/ippTasks/detrend.reject.pro
r8691 r9081 13 13 ## STATE DET_ID ITER TYPE 14 14 15 queueinit DetrendRejectExp 16 if ($?network == 0) 17 $network = 1 18 end 19 if ($?parallel == 0) 20 $parallel = 0 21 end 22 15 23 # select images ready for copy 16 24 # new entries are added to queue DetrendRejectExp 17 25 # compare the new list with the ones already selected 18 task dettool. assess.load26 task dettool.reject.load 19 27 command dettool -residdetrun -simple 20 28 host local … … 22 30 periods -poll 1 23 31 periods -exec 5 24 periods -timeout 30 32 periods -timeout 5 33 34 stdout dettool.reject.log 35 stderr dettool.reject.log 25 36 26 37 # success … … 36 47 $DET_ID = $word:0 37 48 $ITER = $word:1 38 $ TYPE= $word:239 queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $ TYPE"49 $DET_TYPE = $word:2 50 queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE" 40 51 end 41 52 … … 61 72 62 73 # copy new images, sending job to desired host 63 task dettool.re sid.process74 task dettool.reject.process 64 75 periods -poll 0.5 65 76 periods -exec 1.0 66 periods -timeout 577 periods -timeout 30 67 78 68 79 task.exec … … 84 95 ## NOTE: the queue has the STATE prepended 85 96 list word -split $line 86 $DET_ID = $word:1 87 $ITER = $word:2 88 $TYPE = $word:3 97 $DET_ID = $word:0 98 $ITER = $word:1 99 $DET_TYPE = $word:2 100 101 # I'd like to add CAMERA to the log file... 102 stdout isp.$DET_TYPE.$DET_ID.$ITER.log 103 stderr isp.$DET_TYPE.$DET_ID.$ITER.log 89 104 90 105 # specify choice of remote host: 91 host anyhost 106 if ($parallel) 107 host anyhost 108 else 109 host local 110 end 92 111 93 112 # create example job options as a demonstration 94 113 options "$line" 95 # detrend_reject_exp.pl 1 0 bias 96 # detrend_reject_exp.pl DET_ID ITER TYPE 97 command detrend_reject_imfile.pl $DET_ID $ITER $TYPE 98 end 114 echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE 115 command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE 116 end 99 117 100 118 # success … … 110 128 # operation times out? 111 129 task.exit timeout 112 queuepush DetrendRejectExp -replace -key 1:2 " FAIL$options:0"130 queuepush DetrendRejectExp -replace -key 1:2 "TIMEOUT $options:0" 113 131 end 114 132 end -
trunk/ippTasks/detrend.resid.pro
r8691 r9081 13 13 ## the DetrendResidImfiles queue contains: 14 14 ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN 15 16 queueinit DetrendResidImfiles 17 queueinit DetrendResidExp 18 if ($?network == 0) 19 $network = 1 20 end 21 if ($?parallel == 0) 22 $parallel = 0 23 end 15 24 16 25 # select images ready for copy … … 23 32 periods -poll 1 24 33 periods -exec 5 25 periods -timeout 30 34 periods -timeout 5 35 36 stdout dettool.resid.log 37 stderr dettool.resid.log 26 38 27 39 # success … … 38 50 list word -split $line 39 51 $ITER = $word:0 40 $ TYPE= $word:152 $DET_TYPE = $word:1 41 53 $DET_ID = $word:2 42 54 $EXP_KEY = $word:3 43 55 $CLASS_ID = $word:4 44 56 $URI_IN = $word:5 45 queuepush DetrendResidImfiles -uniq -key 1:3:4:5 "NEW $ITER $TYPE $DET_ID $EXP_KEY $CLASS_ID $URI_IN" 57 ## XXX need to include the detID and iter in the key (if we have two running at the same time...) 58 ## Also needed to be in the output file (or path) 59 queuepush DetrendResidImfiles -uniq -key 1:3:4:5 "NEW $ITER $DET_TYPE $DET_ID $EXP_KEY $CLASS_ID $URI_IN" 46 60 end 47 61 … … 69 83 task dettool.resid.process 70 84 periods -poll 0.5 71 periods -exec 1.072 periods -timeout 585 periods -exec 5.0 86 periods -timeout 15 73 87 74 88 task.exec … … 90 104 ## NOTE: the queue has the STATE prepended 91 105 list word -split $line 92 $ITER = $word:1 93 $TYPE = $word:2 94 $DET_ID = $word:3 95 $EXP_KEY = $word:4 96 $CLASS_ID = $word:5 97 $URI_IN = $word:6 106 $ITER = $word:0 107 $DET_TYPE = $word:1 108 $DET_ID = $word:2 109 $EXP_KEY = $word:3 110 $CLASS_ID = $word:4 111 $URI_IN = $word:5 112 113 stdout $EXP_KEY.detresid.$DET_ID.$ITER.$CLASS_ID.log 114 stderr $EXP_KEY.detresid.$DET_ID.$ITER.$CLASS_ID.log 98 115 99 116 # specify choice of remote host: 100 host anyhost 117 if ($parallel) 118 host anyhost 119 else 120 host local 121 end 101 122 102 123 # create example job options as a demonstration 103 124 options "$line" 104 125 # detrend_create_resid.pl DET_ID ITER EXP_KEY CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_KEY ???) 105 $URL_DET = $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit 106 command detrend_create_resid.pl $DET_ID $ITER $EXP_KEY $CLASS_ID $TYPE $URI_DET $URL_IN $EXP_KEY 126 $URL_DET = isp.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits 127 # $URL_DET = $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits 128 echo command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN 129 command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN 107 130 end 108 131 … … 119 142 # operation times out? 120 143 task.exit timeout 121 queuepush DetrendResidImfiles -replace -key 1:3:4:5 "FAIL $options:0" 122 end 123 end 144 queuepush DetrendResidImfiles -replace -key 1:3:4:5 "TIMEOUT $options:0" 145 end 146 end 147 148 # select images ready for copy 149 # new entries are added to queue DetrendResidExp 150 # compare the new list with the ones already selected 151 task dettool.residexp.load 152 command dettool -toresidexp -simple 153 host local 154 155 periods -poll 1 156 periods -exec 5 157 periods -timeout 5 158 159 stdout dettool.residexp.log 160 stderr dettool.residexp.log 161 162 # success 163 task.exit 0 164 # XXX is it necessary for these to be local? 165 # drop after this task macro is done? 166 local i Nqueue 167 168 # compare output with newImage queue 169 # only add entries which don't exist in queue 170 queuesize stdout -var Nqueue 171 for i 0 $Nqueue 172 queuepop stdout -var line 173 list word -split $line 174 $DET_ID = $word:0 175 $ITER = $word:1 176 $DET_TYPE = $word:2 177 $EXP_KEY = $word:3 178 $INCLUDE = $word:4 179 queuepush DetrendResidExp -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_KEY $INCLUDE" 180 end 181 182 # delete existing entries which are DONE 183 queuedrop DetrendResidExp -key 0 DONE 184 end 185 186 # locked list 187 task.exit 1 188 echo "new.images: exec failure" 189 end 190 191 # default exit status 192 task.exit default 193 echo "new.images: unknown exit status: $EXIT" 194 end 195 196 # operation times out? 197 task.exit timeout 198 echo "new.images: timeout" 199 end 200 end 201 202 # copy new images, sending job to desired host 203 task dettool.residexp.process 204 periods -poll 0.5 205 periods -exec 5.0 206 periods -timeout 15 207 208 task.exec 209 queuesize DetrendResidExp -var N 210 if ($N == 0) break 211 if ($network == 0) break 212 213 # look for new images on the internal DetrendResidExp queue 214 # caution with these 'if' statements: syntax errors 215 # will make the task fail without given a good status 216 queuepop DetrendResidExp -var line -key 0 NEW 217 if ("$line" == "NULL") break 218 219 strpop line state 220 queuepush DetrendResidExp -replace -key 1:2:4 "RUN $line" 221 222 ## the DetrendResidExp queue contains: 223 ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN 224 ## NOTE: the queue has the STATE prepended 225 list word -split $line 226 $DET_ID = $word:0 227 $ITER = $word:1 228 $DET_TYPE = $word:2 229 $EXP_KEY = $word:3 230 $INCLUDE = $word:4 231 232 stdout $EXP_KEY.detresid.$DET_ID.$ITER.log 233 stderr $EXP_KEY.detresid.$DET_ID.$ITER.log 234 235 # specify choice of remote host: 236 if ($parallel) 237 host anyhost 238 else 239 host local 240 end 241 242 # create example job options as a demonstration 243 options "$line" 244 echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --det_type $DET_TYPE 245 command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --det_type $DET_TYPE 246 end 247 248 # success 249 task.exit 0 250 queuepush DetrendResidExp -replace -key 1:2:4 "DONE $options:0" 251 end 252 253 # default exit status 254 task.exit default 255 queuepush DetrendResidExp -replace -key 1:2:4 "FAIL $options:0" 256 end 257 258 # operation times out? 259 task.exit timeout 260 queuepush DetrendResidExp -replace -key 1:2:4 "TIMEOUT $options:0" 261 end 262 end -
trunk/ippTasks/detrend.stack.pro
r8691 r9081 13 13 ## these tasks use the queue DetrendImfilesToProcess 14 14 ## the DetrendClassIDtoStack queue contains: 15 ## STATE DET_ID ITER TYPE CLASS_ID 15 ## STATE DET_ID ITER TYPE CLASS_ID CAMERA 16 17 queueinit DetrendClassIDtoStack 18 if ($?network == 0) 19 $network = 1 20 end 21 if ($?parallel == 0) 22 $parallel = 0 23 end 16 24 17 25 # select images ready for copy … … 24 32 periods -poll 1 25 33 periods -exec 5 26 periods -timeout 30 34 periods -timeout 5 35 36 stdout dettool.stack.log 37 stderr dettool.stack.log 27 38 28 39 # success … … 42 53 $TYPE = $word:2 43 54 $CLASS_ID = $word:3 44 queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID" 55 $CAMERA = $word:4 56 echo queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA" 57 queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA" 45 58 end 46 59 … … 69 82 periods -poll 0.5 70 83 periods -exec 1.0 71 periods -timeout 584 periods -timeout 60 72 85 73 86 task.exec … … 89 102 # STATE DET_ID ITER TYPE CLASS_ID 90 103 ## NOTE: the queue has the STATE prepended 104 # use local variables 91 105 list word -split $line 92 $DET_ID = $word:1 93 $ITER = $word:2 94 $TYPE = $word:3 95 $CLASS_ID = $word:4 106 $DET_ID = $word:0 107 $ITER = $word:1 108 $TYPE = $word:2 109 $CLASS_ID = $word:3 110 $CAMERA = $word:3 111 112 stdout $CAMERA.$TYPE.$DET_ID.$ITER.$CLASS_ID.log 113 stderr $CAMERA.$TYPE.$DET_ID.$ITER.$CLASS_ID.log 96 114 97 115 # specify choice of remote host: 98 host anyhost 116 if ($parallel) 117 host anyhost 118 else 119 host local 120 end 99 121 100 122 # create example job options as a demonstration 101 123 options "$line" 102 # detrend_stack.pl 1 0 mc bias 103 command detrend_stack.pl $DET_ID $ITER $CLASS_ID $TYPE 124 # detrend_stack.pl --det_id 1 --iteration 0 --class_id isp --det_type bias --camera ISP || exit 1 125 echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $TYPE --camera $CAMERA 126 command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $TYPE --camera $CAMERA 104 127 end 105 128 … … 116 139 # operation times out? 117 140 task.exit timeout 118 queuepush DetrendClassIDtoStack -replace -key 1:2:4 " FAIL$options:0"141 queuepush DetrendClassIDtoStack -replace -key 1:2:4 "TIMEOUT $options:0" 119 142 end 120 143 end -
trunk/ippTasks/phase0.pro
r9032 r9081 11 11 if ($?network == 0) 12 12 $network = 1 13 end 14 if ($?parallel == 0) 15 $parallel = 0 13 16 end 14 17 … … 59 62 task phase0.imfile.process 60 63 periods -poll 0.5 61 periods -exec 1 .062 periods -timeout 564 periods -exec 1 65 periods -timeout 60 63 66 64 67 task.exec … … 80 83 list word -split $line 81 84 82 # specify choice of remote host: 83 # 'anyhost' specifies random remote host 84 # if we needed to specify a host explicitly, use a command like 85 # one of the following. the -required forces execution on the 86 # named host, and blocks the task until the host is available 87 # host kiawe 88 # host alala -required 89 ## XXX need to determine the appropriate host 90 # host anyhost 91 host local 92 echo "running phase0.imfile.process" 93 94 echo "setting stdout, stderr: $word:1.$word:3.log" 85 # specify choice of remote host:(need to choose based on chips) 86 if ($parallel) 87 host anyhost 88 else 89 host local 90 end 91 95 92 # do something better with this output target 96 93 stdout $word:0.$word:2.log … … 98 95 99 96 # create the command line 100 echo "setting options $line"101 97 options "$line" 102 103 98 echo command phase0imfile.pl --exp_id $word:0 --class_id $word:2 --uri $word:3 104 99 command phase0imfile.pl --exp_id $word:0 --class_id $word:2 --uri $word:3 … … 113 108 # default exit status 114 109 task.exit default 110 echo "failure for phase0imfile.pl: $options:0" 115 111 queuepush Phase0PendingImfiles -replace -key 1 "FAIL $options:0" 116 112 end … … 118 114 # operation times out? 119 115 task.exit timeout 120 queuepush Phase0PendingImfiles -replace -key 1 "FAIL $options:0" 116 echo "timeout for phase0imfile.pl: $options:0" 117 queuepush Phase0PendingImfiles -replace -key 1 "TIMEOUT $options:0" 121 118 end 122 119 end … … 139 136 stdout phase0.exp.log 140 137 stderr phase0.exp.log 141 142 task.exec143 break144 end145 138 146 139 # success … … 196 189 197 190 # do something better with this output target 198 stdout $word:1.log 199 stderr $word:1.log 200 201 # specify choice of remote host: 202 # 'anyhost' specifies random remote host 203 # if we needed to specify a host explicitly, use a command like 204 # one of the following. the -required forces execution on the 205 # named host, and blocks the task until the host is available 206 # host kiawe 207 # host alala -required 208 ## XXX need to determine the appropriate host 209 host local 210 # host anyhost 191 stdout $word:0.log 192 stderr $word:0.log 193 194 # specify choice of remote host:(need to choose based on chips) 195 if ($parallel) 196 host anyhost 197 else 198 host local 199 end 211 200 212 201 # create the command line 213 202 options "$line" 214 command phase0exp.pl $word:1 203 echo command phase0exp.pl $word:0 204 command phase0exp.pl $word:0 215 205 end 216 206 … … 228 218 # operation times out? 229 219 task.exit timeout 230 queuepush Phase0PendingExposures -replace -key 1 "FAIL $options:0" 231 end 232 end 220 queuepush Phase0PendingExposures -replace -key 1 "TIMEOUT $options:0" 221 end 222 end 223 224 ## XXX add a global path to output files
Note:
See TracChangeset
for help on using the changeset viewer.
