Changeset 11011 for branches/eam_branch_00/ippTasks/detrend.process.pro
- Timestamp:
- Jan 10, 2007, 12:12:35 PM (19 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branch_00/ippTasks/detrend.process.pro
r10745 r11011 1 1 ## this file contains the tasks for running the detrend processing stage 2 3 # query for pending imfiles to process: 4 # dettool -raw -simple 5 # 1 bias 850131b fpa mc 850131b.fits BIAS r.MP9601 1.656000 5.496118 0.349071 0.000000 1155.934692 107.037283 46.214947 37.160000 105.480000 -133.067001 0.000000 6 # this returns many columns for manual operations. we strip all but the first few: 7 8 # process a single imfile 9 # detrend_process.pl 1 850131b mc bias 850131b.fits 850131b 10 11 ## these tasks use the queue DetrendProcessImfiles 12 ## the DetrendProcessImfiles queue contains: 13 ## STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI 14 15 queueinit DetrendProcessImfiles 16 queueinit DetrendProcessExposures 2 ## these tasks use the books DetrendProcessImfiles and DetrendProcessExposures 3 17 4 if ($?network == 0) 18 5 $network = 1 … … 21 8 $parallel = 0 22 9 end 23 24 $DetrendProcessImfileFail = 025 $DetrendProcessExpFail = 026 27 10 if ($?VERBOSE == 0) 28 11 echo "VERBOSE not defined: load pantasks.pro first" … … 33 16 break 34 17 end 18 35 19 $LOGSUBDIR = $LOGDIR/detrend 36 20 exec mkdir -p $LOGSUBDIR 37 21 22 book init DetrendProcessImfiles 23 book init DetrendProcessExposures 24 38 25 macro detproc.status 39 queueprint DetrendProcessImfiles 40 queueprint DetrendProcessExposures 41 echo "DetrendProcessImfileFail: $DetrendProcessImfileFail" 42 echo "DetrendProcessExpFail: $DetrendProcessExpFail" 26 book listbook DetrendProcessImfiles 27 book listbook DetrendProcessExposures 43 28 end 44 29 … … 57 42 58 43 task.exec 59 command dettool -toprocessedimfile - simple -limit {$DetrendProcessImfileFail + 20}44 command dettool -toprocessedimfile -limit 20 60 45 end 61 46 62 47 # success 63 48 task.exit 0 64 # XXX is it necessary for these to be local? 65 # drop after this task macro is done? 66 local i Nqueue 67 68 # compare output with newImage queue 69 # only add entries which don't exist in queue 70 queuesize stdout -var Nqueue 71 for i 0 $Nqueue 72 queuepop stdout -var line 73 if ($VERBOSE > 2) 74 echo $line 75 end 76 list word -split $line 77 $DET_ID = $word:0 78 $DET_TYPE = $word:1 79 $EXP_TAG = $word:2 80 $CLASS = $word:3 81 $CLASS_ID = $word:4 82 $URI = $word:5 83 $CAMERA = $word:21 84 queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $DET_TYPE $EXP_TAG $CLASS $CLASS_ID $URI $CAMERA" 49 # convert 'stdout' to book format 50 ipptool2book stdout DetrendProcessImfiles -key det_id:exp_tag:class_id -uniq 51 if ($VERBOSE > 2) 52 book listbook DetrendProcessImfiles 85 53 end 86 54 87 55 # delete existing entries which are DONE 88 queuedrop DetrendProcessImfiles -key 0DONE56 book delpage DetrendProcessImfiles -key state DONE 89 57 end 90 58 … … 100 68 end 101 69 102 # copy new images, sending job to desired host70 # run detrend_process_imfile.ps on pending images 103 71 task dettool.process.run 104 72 periods -poll $runpoll … … 107 75 108 76 task.exec 109 queuesizeDetrendProcessImfiles -var N77 book npages DetrendProcessImfiles -var N 110 78 if ($N == 0) break 111 79 if ($network == 0) break 112 80 113 # look for new images on the internal DetrendProcessImfiles queue 114 # caution with these 'if' statements: syntax errors 115 # will make the task fail without given a good status 116 queuepop DetrendProcessImfiles -var line -key 0 NEW 117 if ("$line" == "NULL") break 118 119 strpop line state 120 queuepush DetrendProcessImfiles -replace -key 1:3:5 "RUN $line" 121 122 ## the DetrendProcessImfiles queue contains: 123 # STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI 124 ## NOTE: the queue has the STATE prepended 125 list word -split $line 126 $DET_ID = $word:0 127 $DET_TYPE = $word:1 128 $EXP_TAG = $word:2 129 $CLASS = $word:3 130 $CLASS_ID = $word:4 131 $URI = $word:5 132 $CAMERA = $word:6 81 # look for new images in DetrendProcessImfiles queue 82 book getpage DetrendProcessImfiles 0 -var pageName -key state NULL 83 if ($pageName == NULL) break 84 85 book setword DetrendProcessImfiles $pageName state RUN 86 book getword DetrendProcessImfiles $pageName det_id -var DET_ID 87 book getword DetrendProcessImfiles $pageName det_type -var DET_TYPE 88 book getword DetrendProcessImfiles $pageName exp_tag -var EXP_TAG 89 book getword DetrendProcessImfiles $pageName class -var CLASS 90 book getword DetrendProcessImfiles $pageName class_id -var CLASS_ID 91 book getword DetrendProcessImfiles $pageName uri -var URI 92 book getword DetrendProcessImfiles $pageName camera -var CAMERA 93 94 # specify choice of remote host: 95 if ($parallel) 96 host anyhost 97 else 98 host local 99 end 133 100 134 101 $word = `basename $EXP_TAG | tr '.' ' '` … … 139 106 exec mkdir -p $LOGSUBDIR/$base 140 107 108 # save the pageName for future reference below 109 options $pageName 110 111 # create example job options as a demonstration 112 setcommand detrend_process_imfile.pl --det_id $DET_ID --det_type $DET_TYPE --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --input_uri $URI --camera $CAMERA 113 end 114 115 # success 116 task.exit 0 117 book setword DetrendProcessImfiles $options:0 state DONE 118 end 119 120 # default exit status 121 task.exit default 122 showcommand failure 123 book setword DetrendProcessImfiles $options:0 state FAIL 124 end 125 126 # operation times out? 127 task.exit timeout 128 showcommand timeout 129 book setword DetrendProcessImfiles $options:0 state TIMEOUT 130 end 131 end 132 133 # select images ready for copy 134 # new entries are added to queue DetrendProcessExposures 135 # compare the new list with the ones already selected 136 task dettool.processexp.load 137 host local 138 139 periods -poll $loadpoll 140 periods -exec $loadexec 141 periods -timeout 30 142 143 stdout $LOGSUBDIR/detproc.exp.log 144 stderr $LOGSUBDIR/detproc.exp.log 145 146 task.exec 147 command dettool -toprocessedexp -limit 20 148 end 149 150 # success 151 task.exit 0 152 # convert 'stdout' to book format 153 ipptool2book stdout DetrendProcessExposures -key det_id:iter:exp_tag -uniq 154 if ($VERBOSE > 2) 155 book listbook DetrendProcessExposures 156 end 157 158 # delete existing entries which are DONE 159 book delpage DetrendProcessExposures -key state DONE 160 end 161 162 # error 163 task.exit default 164 showcommand failure 165 end 166 167 # operation times out? 168 task.exit timeout 169 showcommand timeout 170 end 171 end 172 173 # copy new images, sending job to desired host 174 task dettool.processexp.run 175 periods -poll $runpoll 176 periods -exec $runexec 177 periods -timeout 30 178 179 task.exec 180 book npages DetrendProcessExposures -var N 181 if ($N == 0) break 182 if ($network == 0) break 183 184 # look for new exposures in DetrendProcessExposures 185 book getpage DetrendProcessExposures 0 -var pageName -key state NULL 186 if ($pageName == NULL) break 187 188 book setword DetrendProcessExposures $pageName state RUN 189 book getword DetrendProcessExposures $pageName det_id -var DET_ID 190 book getword DetrendProcessExposures $pageName iter -var ITER 191 book getword DetrendProcessExposures $pageName det_type -var DET_TYPE 192 book getword DetrendProcessExposures $pageName exp_tag -var EXP_TAG 193 book getword DetrendProcessExposures $pageName camera -var CAMERA 194 141 195 # specify choice of remote host: 142 196 if ($parallel) … … 146 200 end 147 201 148 # create example job options as a demonstration149 options "$line"150 if ($VERBOSE > 1)151 echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA152 end153 command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA154 end155 156 # success157 task.exit 0158 queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0"159 # detrend_process_imfile.pl --force_exist --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA160 end161 162 # default exit status163 task.exit default164 showcommand failure165 queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0"166 $DetrendProcessImfileFail ++167 end168 169 # operation times out?170 task.exit timeout171 showcommand timeout172 queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0"173 $DetrendProcessImfileFail ++174 end175 end176 177 # select images ready for copy178 # new entries are added to queue DetrendProcessExposures179 # compare the new list with the ones already selected180 task dettool.processexp.load181 host local182 183 periods -poll $loadpoll184 periods -exec $loadexec185 periods -timeout 30186 187 stdout $LOGSUBDIR/detproc.exp.log188 stderr $LOGSUBDIR/detproc.exp.log189 190 task.exec191 command dettool -toprocessedexp -simple -limit {$DetrendProcessExpFail + 20}192 end193 194 # success195 task.exit 0196 # XXX is it necessary for these to be local?197 # drop after this task macro is done?198 local i Nqueue199 200 # compare output with newImage queue201 # only add entries which don't exist in queue202 queuesize stdout -var Nqueue203 for i 0 $Nqueue204 queuepop stdout -var line205 if ($VERBOSE > 2)206 echo $line207 end208 list word -split $line209 $DET_ID = $word:0210 $ITER = $word:1211 $DET_TYPE = $word:2212 $EXP_TAG = $word:3213 $CAMERA = $word:4214 queuepush DetrendProcessExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_TAG $CAMERA"215 end216 217 # delete existing entries which are DONE218 queuedrop DetrendProcessExposures -key 0 DONE219 end220 221 # error222 task.exit default223 showcommand failure224 end225 226 # operation times out?227 task.exit timeout228 showcommand timeout229 end230 end231 232 # copy new images, sending job to desired host233 task dettool.processexp.run234 periods -poll $runpoll235 periods -exec $runexec236 periods -timeout 30237 238 task.exec239 queuesize DetrendProcessExposures -var N240 if ($N == 0) break241 if ($network == 0) break242 243 # look for new images on the internal DetrendProcessExposures queue244 # caution with these 'if' statements: syntax errors245 # will make the task fail without given a good status246 queuepop DetrendProcessExposures -var line -key 0 NEW247 if ("$line" == "NULL") break248 249 strpop line state250 queuepush DetrendProcessExposures -replace -key 1:2:4 "RUN $line"251 252 list word -split $line253 $DET_ID = $word:0254 $ITER = $word:1255 $DET_TYPE = $word:2256 $EXP_TAG = $word:3257 $CAMERA = $word:4258 259 202 # XXX this is still a lame rule 260 203 $word = `basename $EXP_TAG | tr '.' ' '` 261 204 list word -split $word 262 205 $base = $word:0 263 264 206 stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log 265 207 stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log 266 208 exec mkdir -p $LOGSUBDIR/$base 267 209 268 # specify choice of remote host: 269 if ($parallel) 270 host anyhost 271 else 272 host local 273 end 210 # save the pageName for future reference below 211 options $pageName 274 212 275 213 # create example job options as a demonstration 276 options "$line"277 214 if ($VERBOSE > 1) 278 215 echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA … … 283 220 # success 284 221 task.exit 0 285 queuepush DetrendProcessExposures -replace -key 1:2:4 "DONE $options:0"222 book setword DetrendProcessExposures $options:0 state DONE 286 223 end 287 224 … … 289 226 task.exit default 290 227 showcommand failure 291 queuepush DetrendProcessExposures -replace -key 1:2:4 "FAIL $options:0" 292 $DetrendProcessExpFail ++ 228 book setword DetrendProcessExposures $options:0 state FAIL 293 229 end 294 230 … … 296 232 task.exit timeout 297 233 showcommand timeout 298 queuepush DetrendProcessExposures -replace -key 1:2:4 "TIMEOUT $options:0" 299 $DetrendProcessExpFail ++ 300 end 301 end 234 book setword DetrendProcessExposures $options:0 state TIMEOUT 235 end 236 end
Note:
See TracChangeset
for help on using the changeset viewer.
