Changeset 11011
- Timestamp:
- Jan 10, 2007, 12:12:35 PM (19 years ago)
- Location:
- branches/eam_branch_00/ippTasks
- Files:
-
- 9 edited
-
detrend.norm.pro (modified) (15 diffs)
-
detrend.process.pro (modified) (11 diffs)
-
detrend.reject.pro (modified) (9 diffs)
-
detrend.resid.pro (modified) (8 diffs)
-
detrend.stack.pro (modified) (10 diffs)
-
pantasks.pro (modified) (1 diff)
-
phase0.pro (modified) (16 diffs)
-
phase2.pro (modified) (10 diffs)
-
phase3.pro (modified) (9 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branch_00/ippTasks/detrend.norm.pro
r10745 r11011 1 2 ## this file contains the tasks for running the detrend normalization stage 3 4 ## these tasks use the queue DetrendNorm 5 ## the DetrendNorm queue contains: 6 ## STATE DET_ID ITER 7 8 queueinit DetrendNorm 9 queueinit DetrendNormStat 10 queueinit DetrendNormExp 1 ## this file contains the tasks for running the detrend normalization stages 2 11 3 if ($?network == 0) 12 4 $network = 1 … … 15 7 $parallel = 0 16 8 end 17 18 $DetrendNormFail = 019 $DetrendNormStatFail = 020 $DetrendNormExpFail = 021 22 9 if ($?VERBOSE == 0) 23 10 echo "VERBOSE not defined: load pantasks.pro first" … … 28 15 break 29 16 end 17 30 18 $LOGSUBDIR = $LOGDIR/detrend 31 19 exec mkdir -p $LOGSUBDIR 32 20 21 book init DetrendNormStat 22 book init DetrendNorm 23 book init DetrendNormExp 24 33 25 macro detnorm.status 34 queueprint DetrendNorm 35 queueprint DetrendNormStat 36 queueprint DetrendNormExp 37 echo "DetrendNormFail: $DetrendNormFail" 38 echo "DetrendNormStatFail: $DetrendNormStatFail" 39 echo "DetrendNormExpFail: $DetrendNormExpFail" 26 book listbook DetrendNormStat 27 book listbook DetrendNorm 28 book listbook DetrendNormExp 40 29 end 41 30 … … 54 43 55 44 task.exec 56 command dettool -tonormalizedstat -simple -limit {$DetrendNormStatFail + 20}45 command dettool -tonormalizedstat -limit 20 57 46 end 58 47 59 48 # success 60 49 task.exit 0 61 # XXX is it necessary for these to be local? 62 # drop after this task macro is done? 63 local i Nqueue 64 65 # compare output with newImage queue 66 # only add entries which don't exist in queue 67 queuesize stdout -var Nqueue 68 for i 0 $Nqueue 69 queuepop stdout -var line 70 if ($VERBOSE > 2) 71 echo $line 72 end 73 list word -split $line 74 $DET_ID = $word:0 75 $DET_TYPE = $word:1 76 $ITER = $word:2 77 $CAMERA = $word:3 78 queuepush DetrendNormStat -uniq -key 1:3 "NEW $DET_ID $DET_TYPE $ITER $CAMERA" 50 # convert 'stdout' to book format 51 ipptool2book stdout DetrendNormStat -key det_id:iter -uniq 52 if ($VERBOSE > 2) 53 book listbook DetrendNormStat 79 54 end 80 55 81 56 # delete existing entries which are DONE 82 queuedrop DetrendNormStat -key 0 DONE 83 end 84 85 # locked list 57 book delpage DetrendNormStat -key state DONE 58 end 59 86 60 # default exit status 87 61 task.exit default … … 102 76 103 77 task.exec 104 queuesizeDetrendNormStat -var N78 book npages DetrendNormStat -var N 105 79 if ($N == 0) break 106 80 if ($network == 0) break 107 81 108 # look for new images on the internal DetrendNormStat queue 109 # caution with these 'if' statements: syntax errors 110 # will make the task fail without given a good status 111 queuepop DetrendNormStat -var line -key 0 NEW 112 if ("$line" == "NULL") break 113 114 strpop line state 115 queuepush DetrendNormStat -replace -key 1:3 "RUN $line" 116 117 # the DetrendNormStat queue contains: 118 # STATE DET_ID ITER 119 list word -split $line 120 $DET_ID = $word:0 121 $DET_TYPE = $word:1 122 $ITER = $word:2 123 $CAMERA = $word:3 124 125 # XXX add $WORKDIR/$LOG_DIR 126 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 127 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 82 # look for new images in DetrendNormStat 83 book getpage DetrendNormStat 0 -var pageName -key state NULL 84 if ($pageName == NULL) break 85 86 book setword DetrendNormStat $pageName state RUN 87 book getword DetrendNormStat $pageName DET_ID -var DET_ID 88 book getword DetrendNormStat $pageName DET_TYPE -var DET_TYPE 89 book getword DetrendNormStat $pageName ITER -var ITER 90 book getword DetrendNormStat $pageName CAMERA -var CAMERA 128 91 129 92 # specify choice of remote host: … … 134 97 end 135 98 99 # XXX add $WORKDIR/$LOG_DIR 100 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 101 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 102 103 # save the pageName for future reference below 104 options $pageName 105 136 106 # create example job options as a demonstration 137 options "$line"138 107 if ($VERBOSE > 1) 139 108 echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE … … 144 113 # success 145 114 task.exit 0 146 queuepush DetrendNormStat -replace -key 1:3 "DONE $options:0" 147 end 148 149 # default exit status 150 task.exit default 151 showcommand failure 152 queuepush DetrendNormStat -replace -key 1:3 "FAIL $options:0" 153 $DetrendNormStatFail ++ 154 end 155 156 # operation times out? 157 task.exit timeout 158 showcommand timeout 159 queuepush DetrendNormStat -replace -key 1:3 "TIMEOUT $options:0" 160 $DetrendNormStatFail ++ 115 book setword DetrendNormStat $options:0 state DONE 116 end 117 118 # default exit status 119 task.exit default 120 showcommand failure 121 book setword DetrendNormStat $options:0 state FAIL 122 end 123 124 # operation times out? 125 task.exit timeout 126 showcommand timeout 127 book setword DetrendNormStat $options:0 state TIMEOUT 161 128 end 162 129 end 163 130 164 131 # select images ready for copy 165 # new entries are added to queueDetrendNorm132 # new entries are added to DetrendNorm 166 133 # compare the new list with the ones already selected 167 134 task dettool.norm.load … … 176 143 177 144 task.exec 178 command dettool -tonormalize -simple -limit {$DetrendNormFail + 20}145 command dettool -tonormalize -limit 20 179 146 end 180 147 181 148 # success 182 149 task.exit 0 183 # XXX is it necessary for these to be local? 184 # drop after this task macro is done? 185 local i Nqueue 186 187 # compare output with newImage queue 188 # only add entries which don't exist in queue 189 queuesize stdout -var Nqueue 190 for i 0 $Nqueue 191 queuepop stdout -var line 192 if ($VERBOSE > 2) 193 echo $line 194 end 195 list word -split $line 196 $DET_TYPE = $word:0 197 $CAMERA = $word:1 198 $URI = $word:2 199 $DET_ID = $word:3 200 $ITER = $word:4 201 $CLASS_ID = $word:5 202 $NORM = $word:6 203 queuepush DetrendNorm -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM" 150 # convert 'stdout' to book format 151 ipptool2book stdout DetrendNorm -key det_id:iter -uniq 152 if ($VERBOSE > 2) 153 book listbook DetrendNorm 204 154 end 205 155 206 156 # delete existing entries which are DONE 207 queuedrop DetrendNorm -key 0 DONE 208 end 209 210 # locked list 157 book delpage DetrendNorm -key state DONE 158 end 159 211 160 # default exit status 212 161 task.exit default … … 227 176 228 177 task.exec 229 queuesizeDetrendNorm -var N178 book npages DetrendNorm -var N 230 179 if ($N == 0) break 231 180 if ($network == 0) break 232 181 233 # look for new images on the internal DetrendNorm queue 234 # caution with these 'if' statements: syntax errors 235 # will make the task fail without given a good status 236 queuepop DetrendNorm -var line -key 0 NEW 237 if ("$line" == "NULL") break 238 239 strpop line state 240 queuepush DetrendNorm -replace -key 4:5 "RUN $line" 241 242 # the DetrendNorm queue contains: 243 # STATE DET_ID ITER 244 list word -split $line 245 $DET_TYPE = $word:0 246 $CAMERA = $word:1 247 $URI = $word:2 248 $DET_ID = $word:3 249 $ITER = $word:4 250 $CLASS_ID = $word:5 251 $NORM = $word:6 252 253 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 254 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 182 # look for new images in DetrendNorm 183 book getpage DetrendNorm 0 -var pageName -key state NULL 184 if ($pageName == NULL) break 185 186 book setword DetrendNorm $pageName state RUN 187 book getword DetrendNorm $pageName det_type -var DET_TYPE 188 book getword DetrendNorm $pageName camera -var CAMERA 189 book getword DetrendNorm $pageName uri -var URI 190 book getword DetrendNorm $pageName det_id -var DET_ID 191 book getword DetrendNorm $pageName iter -var ITER 192 book getword DetrendNorm $pageName class_id -var CLASS_ID 193 book getword DetrendNorm $pageName norm -var NORM 255 194 256 195 # specify choice of remote host: … … 261 200 end 262 201 202 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 203 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 204 205 # save the pageName for future reference below 206 options $pageName 207 263 208 # create example job options as a demonstration 264 options "$line"265 209 if ($VERBOSE > 1) 266 210 echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE … … 271 215 # success 272 216 task.exit 0 273 queuepush DetrendNorm -replace -key 4:5 "DONE $options:0" 274 end 275 276 # default exit status 277 task.exit default 278 showcommand failure 279 queuepush DetrendNorm -replace -key 4:5 "FAIL $options:0" 280 $DetrendNormFail ++ 281 end 282 283 # operation times out? 284 task.exit timeout 285 showcommand timeout 286 queuepush DetrendNorm -replace -key 4:5 "TIMEOUT $options:0" 287 $DetrendNormFail ++ 217 book setword DetrendNorm $options:0 state DONE 218 end 219 220 # default exit status 221 task.exit default 222 showcommand failure 223 book setword DetrendNorm $options:0 state FAIL 224 end 225 226 # operation times out? 227 task.exit timeout 228 showcommand timeout 229 book setword DetrendNorm $options:0 state TIMEOUT 288 230 end 289 231 end … … 303 245 304 246 task.exec 305 command dettool -tonormalizedexp -simple -limit {$DetrendNormExpFail + 20}247 command dettool -tonormalizedexp -limit 20 306 248 end 307 249 308 250 # success 309 251 task.exit 0 310 # XXX is it necessary for these to be local? 311 # drop after this task macro is done? 312 local i Nqueue 313 314 # compare output with newImage queue 315 # only add entries which don't exist in queue 316 queuesize stdout -var Nqueue 317 for i 0 $Nqueue 318 queuepop stdout -var line 319 if ($VERBOSE > 2) 320 echo $line 321 end 322 list word -split $line 323 $DET_ID = $word:0 324 $ITER = $word:1 325 $DET_TYPE = $word:2 326 $CAMERA = $word:3 327 #$TELESCOPE= $word:4 328 #$EXP_TYPE = $word:5 329 #$IMFILES = $word:6 330 queuepush DetrendNormExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA" 252 # convert 'stdout' to book format 253 ipptool2book stdout DetrendNormExp -key det_id:iter -uniq 254 if ($VERBOSE > 2) 255 book listbook DetrendNormExp 331 256 end 332 257 333 258 # delete existing entries which are DONE 334 queuedrop DetrendNormExp -key 0 DONE 335 end 336 337 # locked list 259 book delpage DetrendNormExp -key state DONE 260 end 261 338 262 # default exit status 339 263 task.exit default … … 354 278 355 279 task.exec 356 queuesize DetrendNormExp -var N 357 280 book npages DetrendNormExp -var N 358 281 if ($N == 0) break 359 282 if ($network == 0) break 360 283 361 # look for new images on the internal DetrendNormExp queue 362 # caution with these 'if' statements: syntax errors 363 # will make the task fail without given a good status 364 queuepop DetrendNormExp -var line -key 0 NEW 365 if ("$line" == "NULL") break 366 367 strpop line state 368 queuepush DetrendNormExp -replace -key 1:2 "RUN $line" 369 370 # the DetrendNormExp queue contains: 371 # STATE DET_ID ITER 372 list word -split $line 373 $DET_ID = $word:0 374 $ITER = $word:1 375 $DET_TYPE = $word:2 376 $CAMERA = $word:3 377 #$TELESCOPE= $word:4 378 #$EXP_TYPE = $word:5 379 #$IMFILES = $word:6 380 381 # XXX add $WORKDIR/$LOG_DIR 382 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 383 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 284 # look for new images in DetrendNormExp 285 book getpage DetrendNormExp 0 -var pageName -key state NULL 286 if ($pageName == NULL) break 287 288 book setword DetrendNormExp $pageName state RUN 289 book getword DetrendNormExp $pageName det_id -var DET_ID 290 book getword DetrendNormExp $pageName iter -var ITER 291 book getword DetrendNormExp $pageName det_type -var DET_TYPE 292 book getword DetrendNormExp $pageName camera -var CAMERA 384 293 385 294 # specify choice of remote host: … … 390 299 end 391 300 301 # XXX add $WORKDIR/$LOG_DIR 302 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 303 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 304 305 # save the pageName for future reference below 306 options $pageName 307 392 308 # create example job options as a demonstration 393 options "$line"394 309 if ($VERBOSE > 1) 395 310 echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE … … 400 315 # success 401 316 task.exit 0 402 queuepush DetrendNormExp -replace -key 1:2 "DONE $options:0" 403 end 404 405 # default exit status 406 task.exit default 407 showcommand failure 408 queuepush DetrendNormExp -replace -key 1:2 "FAIL $options:0" 409 $DetrendNormExpFail ++ 410 end 411 412 # operation times out? 413 task.exit timeout 414 showcommand timeout 415 queuepush DetrendNormExp -replace -key 1:2 "TIMEOUT $options:0" 416 $DetrendNormExpFail ++ 417 end 418 end 317 book setword DetrendNormExp $options:0 state DONE 318 end 319 320 # default exit status 321 task.exit default 322 showcommand failure 323 book setword DetrendNormExp $options:0 state FAIL 324 end 325 326 # operation times out? 327 task.exit timeout 328 showcommand timeout 329 book setword DetrendNormExp $options:0 state TIMEOUT 330 end 331 end -
branches/eam_branch_00/ippTasks/detrend.process.pro
r10745 r11011 1 1 ## this file contains the tasks for running the detrend processing stage 2 3 # query for pending imfiles to process: 4 # dettool -raw -simple 5 # 1 bias 850131b fpa mc 850131b.fits BIAS r.MP9601 1.656000 5.496118 0.349071 0.000000 1155.934692 107.037283 46.214947 37.160000 105.480000 -133.067001 0.000000 6 # this returns many columns for manual operations. we strip all but the first few: 7 8 # process a single imfile 9 # detrend_process.pl 1 850131b mc bias 850131b.fits 850131b 10 11 ## these tasks use the queue DetrendProcessImfiles 12 ## the DetrendProcessImfiles queue contains: 13 ## STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI 14 15 queueinit DetrendProcessImfiles 16 queueinit DetrendProcessExposures 2 ## these tasks use the books DetrendProcessImfiles and DetrendProcessExposures 3 17 4 if ($?network == 0) 18 5 $network = 1 … … 21 8 $parallel = 0 22 9 end 23 24 $DetrendProcessImfileFail = 025 $DetrendProcessExpFail = 026 27 10 if ($?VERBOSE == 0) 28 11 echo "VERBOSE not defined: load pantasks.pro first" … … 33 16 break 34 17 end 18 35 19 $LOGSUBDIR = $LOGDIR/detrend 36 20 exec mkdir -p $LOGSUBDIR 37 21 22 book init DetrendProcessImfiles 23 book init DetrendProcessExposures 24 38 25 macro detproc.status 39 queueprint DetrendProcessImfiles 40 queueprint DetrendProcessExposures 41 echo "DetrendProcessImfileFail: $DetrendProcessImfileFail" 42 echo "DetrendProcessExpFail: $DetrendProcessExpFail" 26 book listbook DetrendProcessImfiles 27 book listbook DetrendProcessExposures 43 28 end 44 29 … … 57 42 58 43 task.exec 59 command dettool -toprocessedimfile - simple -limit {$DetrendProcessImfileFail + 20}44 command dettool -toprocessedimfile -limit 20 60 45 end 61 46 62 47 # success 63 48 task.exit 0 64 # XXX is it necessary for these to be local? 65 # drop after this task macro is done? 66 local i Nqueue 67 68 # compare output with newImage queue 69 # only add entries which don't exist in queue 70 queuesize stdout -var Nqueue 71 for i 0 $Nqueue 72 queuepop stdout -var line 73 if ($VERBOSE > 2) 74 echo $line 75 end 76 list word -split $line 77 $DET_ID = $word:0 78 $DET_TYPE = $word:1 79 $EXP_TAG = $word:2 80 $CLASS = $word:3 81 $CLASS_ID = $word:4 82 $URI = $word:5 83 $CAMERA = $word:21 84 queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $DET_TYPE $EXP_TAG $CLASS $CLASS_ID $URI $CAMERA" 49 # convert 'stdout' to book format 50 ipptool2book stdout DetrendProcessImfiles -key det_id:exp_tag:class_id -uniq 51 if ($VERBOSE > 2) 52 book listbook DetrendProcessImfiles 85 53 end 86 54 87 55 # delete existing entries which are DONE 88 queuedrop DetrendProcessImfiles -key 0DONE56 book delpage DetrendProcessImfiles -key state DONE 89 57 end 90 58 … … 100 68 end 101 69 102 # copy new images, sending job to desired host70 # run detrend_process_imfile.ps on pending images 103 71 task dettool.process.run 104 72 periods -poll $runpoll … … 107 75 108 76 task.exec 109 queuesizeDetrendProcessImfiles -var N77 book npages DetrendProcessImfiles -var N 110 78 if ($N == 0) break 111 79 if ($network == 0) break 112 80 113 # look for new images on the internal DetrendProcessImfiles queue 114 # caution with these 'if' statements: syntax errors 115 # will make the task fail without given a good status 116 queuepop DetrendProcessImfiles -var line -key 0 NEW 117 if ("$line" == "NULL") break 118 119 strpop line state 120 queuepush DetrendProcessImfiles -replace -key 1:3:5 "RUN $line" 121 122 ## the DetrendProcessImfiles queue contains: 123 # STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI 124 ## NOTE: the queue has the STATE prepended 125 list word -split $line 126 $DET_ID = $word:0 127 $DET_TYPE = $word:1 128 $EXP_TAG = $word:2 129 $CLASS = $word:3 130 $CLASS_ID = $word:4 131 $URI = $word:5 132 $CAMERA = $word:6 81 # look for new images in DetrendProcessImfiles queue 82 book getpage DetrendProcessImfiles 0 -var pageName -key state NULL 83 if ($pageName == NULL) break 84 85 book setword DetrendProcessImfiles $pageName state RUN 86 book getword DetrendProcessImfiles $pageName det_id -var DET_ID 87 book getword DetrendProcessImfiles $pageName det_type -var DET_TYPE 88 book getword DetrendProcessImfiles $pageName exp_tag -var EXP_TAG 89 book getword DetrendProcessImfiles $pageName class -var CLASS 90 book getword DetrendProcessImfiles $pageName class_id -var CLASS_ID 91 book getword DetrendProcessImfiles $pageName uri -var URI 92 book getword DetrendProcessImfiles $pageName camera -var CAMERA 93 94 # specify choice of remote host: 95 if ($parallel) 96 host anyhost 97 else 98 host local 99 end 133 100 134 101 $word = `basename $EXP_TAG | tr '.' ' '` … … 139 106 exec mkdir -p $LOGSUBDIR/$base 140 107 108 # save the pageName for future reference below 109 options $pageName 110 111 # create example job options as a demonstration 112 setcommand detrend_process_imfile.pl --det_id $DET_ID --det_type $DET_TYPE --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --input_uri $URI --camera $CAMERA 113 end 114 115 # success 116 task.exit 0 117 book setword DetrendProcessImfiles $options:0 state DONE 118 end 119 120 # default exit status 121 task.exit default 122 showcommand failure 123 book setword DetrendProcessImfiles $options:0 state FAIL 124 end 125 126 # operation times out? 127 task.exit timeout 128 showcommand timeout 129 book setword DetrendProcessImfiles $options:0 state TIMEOUT 130 end 131 end 132 133 # select images ready for copy 134 # new entries are added to queue DetrendProcessExposures 135 # compare the new list with the ones already selected 136 task dettool.processexp.load 137 host local 138 139 periods -poll $loadpoll 140 periods -exec $loadexec 141 periods -timeout 30 142 143 stdout $LOGSUBDIR/detproc.exp.log 144 stderr $LOGSUBDIR/detproc.exp.log 145 146 task.exec 147 command dettool -toprocessedexp -limit 20 148 end 149 150 # success 151 task.exit 0 152 # convert 'stdout' to book format 153 ipptool2book stdout DetrendProcessExposures -key det_id:iter:exp_tag -uniq 154 if ($VERBOSE > 2) 155 book listbook DetrendProcessExposures 156 end 157 158 # delete existing entries which are DONE 159 book delpage DetrendProcessExposures -key state DONE 160 end 161 162 # error 163 task.exit default 164 showcommand failure 165 end 166 167 # operation times out? 168 task.exit timeout 169 showcommand timeout 170 end 171 end 172 173 # copy new images, sending job to desired host 174 task dettool.processexp.run 175 periods -poll $runpoll 176 periods -exec $runexec 177 periods -timeout 30 178 179 task.exec 180 book npages DetrendProcessExposures -var N 181 if ($N == 0) break 182 if ($network == 0) break 183 184 # look for new exposures in DetrendProcessExposures 185 book getpage DetrendProcessExposures 0 -var pageName -key state NULL 186 if ($pageName == NULL) break 187 188 book setword DetrendProcessExposures $pageName state RUN 189 book getword DetrendProcessExposures $pageName det_id -var DET_ID 190 book getword DetrendProcessExposures $pageName iter -var ITER 191 book getword DetrendProcessExposures $pageName det_type -var DET_TYPE 192 book getword DetrendProcessExposures $pageName exp_tag -var EXP_TAG 193 book getword DetrendProcessExposures $pageName camera -var CAMERA 194 141 195 # specify choice of remote host: 142 196 if ($parallel) … … 146 200 end 147 201 148 # create example job options as a demonstration149 options "$line"150 if ($VERBOSE > 1)151 echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA152 end153 command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA154 end155 156 # success157 task.exit 0158 queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0"159 # detrend_process_imfile.pl --force_exist --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA160 end161 162 # default exit status163 task.exit default164 showcommand failure165 queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0"166 $DetrendProcessImfileFail ++167 end168 169 # operation times out?170 task.exit timeout171 showcommand timeout172 queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0"173 $DetrendProcessImfileFail ++174 end175 end176 177 # select images ready for copy178 # new entries are added to queue DetrendProcessExposures179 # compare the new list with the ones already selected180 task dettool.processexp.load181 host local182 183 periods -poll $loadpoll184 periods -exec $loadexec185 periods -timeout 30186 187 stdout $LOGSUBDIR/detproc.exp.log188 stderr $LOGSUBDIR/detproc.exp.log189 190 task.exec191 command dettool -toprocessedexp -simple -limit {$DetrendProcessExpFail + 20}192 end193 194 # success195 task.exit 0196 # XXX is it necessary for these to be local?197 # drop after this task macro is done?198 local i Nqueue199 200 # compare output with newImage queue201 # only add entries which don't exist in queue202 queuesize stdout -var Nqueue203 for i 0 $Nqueue204 queuepop stdout -var line205 if ($VERBOSE > 2)206 echo $line207 end208 list word -split $line209 $DET_ID = $word:0210 $ITER = $word:1211 $DET_TYPE = $word:2212 $EXP_TAG = $word:3213 $CAMERA = $word:4214 queuepush DetrendProcessExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_TAG $CAMERA"215 end216 217 # delete existing entries which are DONE218 queuedrop DetrendProcessExposures -key 0 DONE219 end220 221 # error222 task.exit default223 showcommand failure224 end225 226 # operation times out?227 task.exit timeout228 showcommand timeout229 end230 end231 232 # copy new images, sending job to desired host233 task dettool.processexp.run234 periods -poll $runpoll235 periods -exec $runexec236 periods -timeout 30237 238 task.exec239 queuesize DetrendProcessExposures -var N240 if ($N == 0) break241 if ($network == 0) break242 243 # look for new images on the internal DetrendProcessExposures queue244 # caution with these 'if' statements: syntax errors245 # will make the task fail without given a good status246 queuepop DetrendProcessExposures -var line -key 0 NEW247 if ("$line" == "NULL") break248 249 strpop line state250 queuepush DetrendProcessExposures -replace -key 1:2:4 "RUN $line"251 252 list word -split $line253 $DET_ID = $word:0254 $ITER = $word:1255 $DET_TYPE = $word:2256 $EXP_TAG = $word:3257 $CAMERA = $word:4258 259 202 # XXX this is still a lame rule 260 203 $word = `basename $EXP_TAG | tr '.' ' '` 261 204 list word -split $word 262 205 $base = $word:0 263 264 206 stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log 265 207 stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log 266 208 exec mkdir -p $LOGSUBDIR/$base 267 209 268 # specify choice of remote host: 269 if ($parallel) 270 host anyhost 271 else 272 host local 273 end 210 # save the pageName for future reference below 211 options $pageName 274 212 275 213 # create example job options as a demonstration 276 options "$line"277 214 if ($VERBOSE > 1) 278 215 echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA … … 283 220 # success 284 221 task.exit 0 285 queuepush DetrendProcessExposures -replace -key 1:2:4 "DONE $options:0"222 book setword DetrendProcessExposures $options:0 state DONE 286 223 end 287 224 … … 289 226 task.exit default 290 227 showcommand failure 291 queuepush DetrendProcessExposures -replace -key 1:2:4 "FAIL $options:0" 292 $DetrendProcessExpFail ++ 228 book setword DetrendProcessExposures $options:0 state FAIL 293 229 end 294 230 … … 296 232 task.exit timeout 297 233 showcommand timeout 298 queuepush DetrendProcessExposures -replace -key 1:2:4 "TIMEOUT $options:0" 299 $DetrendProcessExpFail ++ 300 end 301 end 234 book setword DetrendProcessExposures $options:0 state TIMEOUT 235 end 236 end -
branches/eam_branch_00/ippTasks/detrend.reject.pro
r10745 r11011 1 2 1 ## this file contains the tasks for running the detrend processing stage 3 2 4 # dettool -residdetrun -simple5 # 1 0 bias6 # DET_ID ITER TYPE7 8 # detrend_reject_exp.pl 1 0 bias9 # detrend_reject_exp.pl DET_ID ITER TYPE10 11 ## these tasks use the queue DetrendResidExp12 ## the DetrendRejectExp queue contains:13 ## STATE DET_ID ITER TYPE14 15 queueinit DetrendRejectExp16 3 if ($?network == 0) 17 4 $network = 1 … … 20 7 $parallel = 0 21 8 end 22 23 $DetrendRejectExpFail = 024 25 9 if ($?VERBOSE == 0) 26 10 echo "VERBOSE not defined: load pantasks.pro first" … … 31 15 break 32 16 end 17 33 18 $LOGSUBDIR = $LOGDIR/detrend 34 19 exec mkdir -p $LOGSUBDIR 35 20 21 book init DetrendRejectExp 22 36 23 macro detreject.status 37 queueprint DetrendRejectExp 38 echo "DetrendRejectExpFail: $DetrendRejectExpFail" 24 book listbook DetrendRejectExp 39 25 end 40 26 … … 53 39 54 40 task.exec 55 command dettool -residdetrun - simple -limit {$DetrendRejectExpFail + 20}41 command dettool -residdetrun -limit 20 56 42 end 57 43 58 44 # success 59 45 task.exit 0 60 local i Nqueue 61 62 # compare output with newImage queue 63 # only add entries which don't exist in queue 64 queuesize stdout -var Nqueue 65 for i 0 $Nqueue 66 queuepop stdout -var line 67 if ($VERBOSE > 2) 68 echo $line 69 end 70 list word -split $line 71 $DET_ID = $word:0 72 $ITER = $word:1 73 $DET_TYPE = $word:2 74 $MODE = $word:3 75 $CAMERA = $word:4 76 queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $MODE $CAMERA" 46 # convert 'stdout' to book format 47 ipptool2book stdout DetrendRejectExp -key det_id:iter -uniq 48 if ($VERBOSE > 2) 49 book listbook DetrendRejectExp 77 50 end 78 51 79 52 # delete existing entries which are DONE 80 queuedrop DetrendRejectExp -key 0DONE53 book delpage DetrendRejectExp -key state DONE 81 54 end 82 55 … … 99 72 100 73 task.exec 101 queuesizeDetrendRejectExp -var N74 book npages DetrendRejectExp -var N 102 75 if ($N == 0) break 103 76 if ($network == 0) break 104 77 105 # look for new images on the internal DetrendRejectExp queue 106 # caution with these 'if' statements: syntax errors 107 # will make the task fail without given a good status 108 queuepop DetrendRejectExp -var line -key 0 NEW 109 if ("$line" == "NULL") break 78 # look for new images in DetrendRejectExp 79 book getpage DetrendRejectExp 0 -var pageName -key state NULL 80 if ($pageName == NULL) break 110 81 111 strpop line state 112 queuepush DetrendRejectExp -replace -key 1:2 "RUN $line" 113 114 ## the DetrendRejectExp queue contains: 115 ## STATE DET_ID ITER TYPE 116 ## NOTE: the queue has the STATE prepended 117 list word -split $line 118 $DET_ID = $word:0 119 $ITER = $word:1 120 $DET_TYPE = $word:2 121 $MODE = $word:3 122 $CAMERA = $word:4 123 124 # I'd like to add CAMERA to the log file... 125 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 126 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 82 book setword DetrendRejectExp $pageName state RUN 83 book getword DetrendRejectExp $pageName DET_ID -var DET_ID 84 book getword DetrendRejectExp $pageName ITER -var ITER 85 book getword DetrendRejectExp $pageName DET_TYPE -var DET_TYPE 86 book getword DetrendRejectExp $pageName MODE -var MODE 87 book getword DetrendRejectExp $pageName CAMERA -var CAMERA 127 88 128 89 # specify choice of remote host: … … 133 94 end 134 95 96 # I'd like to add CAMERA to the log file... 97 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 98 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 99 100 # save the pageName for future reference below 101 options $pageName 102 135 103 # create example job options as a demonstration 136 options "$line"137 104 if ($VERBOSE > 1) 138 105 echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA … … 143 110 # success 144 111 task.exit 0 145 queuepush DetrendRejectExp -replace -key 1:2 "DONE $options:0"112 book setword DetrendRejectExp $options:0 state DONE 146 113 end 147 114 … … 149 116 task.exit default 150 117 showcommand failure 151 queuepush DetrendRejectExp -replace -key 1:2 "FAIL $options:0" 152 $DetrendRejectExpFail ++ 118 book setword DetrendRejectExp $options:0 state FAIL 153 119 end 154 120 … … 156 122 task.exit timeout 157 123 showcommand failure 158 queuepush DetrendRejectExp -replace -key 1:2 "TIMEOUT $options:0" 159 $DetrendRejectExpFail ++ 124 book setword DetrendRejectExp $options:0 state TIMEOUT 160 125 end 161 126 end -
branches/eam_branch_00/ippTasks/detrend.resid.pro
r10745 r11011 1 2 1 ## this file contains the tasks for running the detrend processing stage 3 2 4 # dettool -toresidimfile -simple5 # 0 bias 1 850131b mc o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.5655826 # ITER TYPE DET_ID EXP_TAG CLASS_ID URI-IN7 8 # detrend_create_resid.pl 1 0 850131b mc bias bias_mc_1_0.fit o_850131b.mc.fit 850131b9 # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG)10 # URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit11 12 ## these tasks use the queue DetrendResidImfiles13 ## the DetrendResidImfiles queue contains:14 ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN15 16 queueinit DetrendResidImfiles17 queueinit DetrendResidExposures18 3 if ($?network == 0) 19 4 $network = 1 … … 22 7 $parallel = 0 23 8 end 24 25 $DetrendResidImfileFail = 026 $DetrendResidExpFail = 027 28 9 if ($?VERBOSE == 0) 29 10 echo "VERBOSE not defined: load pantasks.pro first" … … 34 15 break 35 16 end 17 36 18 $LOGSUBDIR = $LOGDIR/detrend 37 19 exec mkdir -p $LOGSUBDIR 38 20 21 book init DetrendResidImfiles 22 book init DetrendResidExposures 23 39 24 macro detresid.status 40 queueprint DetrendResidImfiles 41 queueprint DetrendResidExposures 42 echo "DetrendResidImfileFail: $DetrendResidImfileFail" 43 echo "DetrendResidExpFail: $DetrendResidExpFail" 25 book listbook DetrendResidImfiles 26 book listbook DetrendResidExposures 44 27 end 45 28 … … 58 41 59 42 task.exec 60 command dettool -toresidimfile - simple -limit {$DetrendResidImfileFail + 20}43 command dettool -toresidimfile -limit 20 61 44 end 62 45 63 46 # success 64 47 task.exit 0 65 # XXX is it necessary for these to be local? 66 # drop after this task macro is done? 67 local i Nqueue 68 69 # compare output with newImage queue 70 # only add entries which don't exist in queue 71 queuesize stdout -var Nqueue 72 for i 0 $Nqueue 73 queuepop stdout -var line 74 if ($VERBOSE > 2) 75 echo $line 76 end 77 list word -split $line 78 $DET_ID = $word:0 79 $ITER = $word:1 80 $DET_TYPE = $word:2 81 $MODE = $word:3 82 $EXP_TAG = $word:4 83 $CLASS_ID = $word:5 84 $URI = $word:6 85 $DET_URI = $word:7 86 $CAMERA = $word:8 87 ## must include the detID and iter in the key (if we have two running at the same time...) 88 ## Also needed to be in the output file (or path) 89 queuepush DetrendResidImfiles -uniq -key 1:2:5:6 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $CLASS_ID $URI $DET_URI $CAMERA" 48 # convert 'stdout' to book format 49 ipptool2book stdout DetrendResidImfiles -key det_id:iter:exp_tag:class_id -uniq 50 if ($VERBOSE > 2) 51 book listbook DetrendResidImfiles 90 52 end 91 53 92 54 # delete existing entries which are DONE 93 queuedrop DetrendResidImfiles -key 0 DONE 94 end 95 96 # locked list 55 book delpage DetrendResidImfiles -key state DONE 56 end 57 97 58 # default exit status 98 59 task.exit default … … 113 74 114 75 task.exec 115 queuesizeDetrendResidImfiles -var N76 book npages DetrendResidImfiles -var N 116 77 if ($N == 0) break 117 78 if ($network == 0) break 118 79 119 # look for new images on the internal DetrendResidImfiles queue 120 # caution with these 'if' statements: syntax errors 121 # will make the task fail without given a good status 122 queuepop DetrendResidImfiles -var line -key 0 NEW 123 if ("$line" == "NULL") break 124 125 strpop line state 126 queuepush DetrendResidImfiles -replace -key 1:2:5:6 "RUN $line" 127 128 ## the DetrendResidImfiles queue contains: 129 ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN 130 ## NOTE: the queue has the STATE prepended 131 list word -split $line 132 $DET_ID = $word:0 133 $ITER = $word:1 134 $DET_TYPE = $word:2 135 $MODE = $word:3 136 $EXP_TAG = $word:4 137 $CLASS_ID = $word:5 138 $URI = $word:6 139 $DET_URI = $word:7 140 $CAMERA = $word:8 80 # look for new images in DetrendResidImfiles 81 book getpage DetrendResidImfiles 0 -var pageName -key state NULL 82 if ($pageName == NULL) break 83 84 book setword DetrendResidImfiles $pageName state RUN 85 book getword DetrendResidImfiles $pageName DET_ID -var DET_ID 86 book getword DetrendResidImfiles $pageName ITER -var ITER 87 book getword DetrendResidImfiles $pageName DET_TYPE -var DET_TYPE 88 book getword DetrendResidImfiles $pageName MODE -var MODE 89 book getword DetrendResidImfiles $pageName EXP_TAG -var EXP_TAG 90 book getword DetrendResidImfiles $pageName CLASS_ID -var CLASS_ID 91 book getword DetrendResidImfiles $pageName URI -var URI 92 book getword DetrendResidImfiles $pageName DET_URI -var DET_URI 93 book getword DetrendResidImfiles $pageName CAMERA -var CAMERA 94 95 # specify choice of remote host: 96 if ($parallel) 97 host anyhost 98 else 99 host local 100 end 141 101 142 102 $word = `basename $EXP_TAG | tr '.' ' '` … … 147 107 exec mkdir -p $LOGSUBDIR/$base 148 108 109 # save the pageName for future reference below 110 options $pageName 111 112 # create command 113 if ($VERBOSE > 1) 114 echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE 115 end 116 command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE 117 end 118 119 # success 120 task.exit 0 121 book setword DetrendResidImfiles $options:0 state DONE 122 end 123 124 # default exit status 125 task.exit default 126 showcommand failure 127 book setword DetrendResidImfiles $options:0 state FAIL 128 end 129 130 # operation times out? 131 task.exit timeout 132 showcommand timeout 133 book setword DetrendResidImfiles $options:0 state TIMEOUT 134 end 135 end 136 137 # select images ready for copy 138 # new entries are added to DetrendResidExposures 139 # compare the new list with the ones already selected 140 task dettool.residexp.load 141 host local 142 143 periods -poll $loadpoll 144 periods -exec $loadexec 145 periods -timeout 30 146 147 stdout $LOGSUBDIR/detresid.exp.log 148 stderr $LOGSUBDIR/detresid.exp.log 149 150 task.exec 151 command dettool -toresidexp -limit 20 152 end 153 154 # success 155 task.exit 0 156 # convert 'stdout' to book format 157 ipptool2book stdout DetrendResidExposures -key det_id:iter:exp_tag -uniq 158 if ($VERBOSE > 2) 159 book listbook DetrendResidExposures 160 end 161 162 # delete existing entries which are DONE 163 book delpage DetrendResidExposures -key state DONE 164 end 165 166 # default exit status 167 task.exit default 168 showcommand failure 169 end 170 171 # operation times out? 172 task.exit timeout 173 showcommand timeout 174 end 175 end 176 177 # copy new images, sending job to desired host 178 task dettool.residexp.run 179 periods -poll $runpoll 180 periods -exec $runexec 181 periods -timeout 30 182 183 task.exec 184 book npages DetrendResidExposures -var N 185 if ($N == 0) break 186 if ($network == 0) break 187 188 # look for new images in DetrendResidExposures 189 book getpage DetrendResidExposures 0 -var pageName -key state NULL 190 if ($pageName == NULL) break 191 192 book setword DetrendResidExposures $pageName state RUN 193 book getword DetrendResidExposures $pageName DET_ID -var DET_ID 194 book getword DetrendResidExposures $pageName ITER -var ITER 195 book getword DetrendResidExposures $pageName DET_TYPE -var DET_TYPE 196 book getword DetrendResidExposures $pageName MODE -var MODE 197 book getword DetrendResidExposures $pageName EXP_TAG -var EXP_TAG 198 book getword DetrendResidExposures $pageName INCLUDE -var INCLUDE 199 book getword DetrendResidExposures $pageName CAMERA -var CAMERA 200 149 201 # specify choice of remote host: 150 202 if ($parallel) … … 154 206 end 155 207 156 # create example job options as a demonstration157 options "$line"158 # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???)159 # need to parse the subdir path for the detrend URL160 # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits161 echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE162 command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE163 end164 165 # success166 task.exit 0167 queuepush DetrendResidImfiles -replace -key 1:2:5:6 "DONE $options:0"168 end169 170 # default exit status171 task.exit default172 showcommand failure173 queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0"174 $DetrendResidImfileFail ++175 end176 177 # operation times out?178 task.exit timeout179 showcommand timeout180 queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0"181 $DetrendResidImfileFail ++182 end183 end184 185 # select images ready for copy186 # new entries are added to queue DetrendResidExposures187 # compare the new list with the ones already selected188 task dettool.residexp.load189 host local190 191 periods -poll $loadpoll192 periods -exec $loadexec193 periods -timeout 30194 195 stdout $LOGSUBDIR/detresid.exp.log196 stderr $LOGSUBDIR/detresid.exp.log197 198 task.exec199 command dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20}200 end201 202 # success203 task.exit 0204 # XXX is it necessary for these to be local?205 # drop after this task macro is done?206 local i Nqueue207 208 # compare output with newImage queue209 # only add entries which don't exist in queue210 queuesize stdout -var Nqueue211 for i 0 $Nqueue212 queuepop stdout -var line213 if ($VERBOSE > 2)214 echo $line215 end216 list word -split $line217 $DET_ID = $word:0218 $ITER = $word:1219 $DET_TYPE = $word:2220 $MODE = $word:3221 $EXP_TAG = $word:4222 $INCLUDE = $word:5223 $CAMERA = $word:6224 queuepush DetrendResidExposures -uniq -key 1:2:5 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $INCLUDE $CAMERA"225 end226 227 # delete existing entries which are DONE228 queuedrop DetrendResidExposures -key 0 DONE229 end230 231 # locked list232 # default exit status233 task.exit default234 showcommand failure235 end236 237 # operation times out?238 task.exit timeout239 showcommand timeout240 end241 end242 243 # copy new images, sending job to desired host244 task dettool.residexp.run245 periods -poll $runpoll246 periods -exec $runexec247 periods -timeout 30248 249 task.exec250 queuesize DetrendResidExposures -var N251 if ($N == 0) break252 if ($network == 0) break253 254 # look for new images on the internal DetrendResidExposures queue255 # caution with these 'if' statements: syntax errors256 # will make the task fail without given a good status257 queuepop DetrendResidExposures -var line -key 0 NEW258 if ("$line" == "NULL") break259 260 strpop line state261 queuepush DetrendResidExposures -replace -key 1:2:5 "RUN $line"262 263 ## the DetrendResidExposures queue contains:264 ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN265 ## NOTE: the queue has the STATE prepended266 list word -split $line267 $DET_ID = $word:0268 $ITER = $word:1269 $DET_TYPE = $word:2270 $MODE = $word:3271 $EXP_TAG = $word:4272 $INCLUDE = $word:5273 $CAMERA = $word:6274 275 208 # XXX this is still a lame rule 276 209 $word = `basename $EXP_TAG | tr '.' ' '` 277 210 list word -split $word 278 211 $base = $word:0 279 280 212 stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log 281 213 stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log 282 214 exec mkdir -p $LOGSUBDIR/$base 283 215 284 # specify choice of remote host: 285 if ($parallel) 286 host anyhost 287 else 288 host local 289 end 216 # save the pageName for future reference below 217 options $pageName 290 218 291 219 # create example job options as a demonstration 292 options "$line"293 220 if ($VERBOSE > 1) 294 221 echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA … … 299 226 # success 300 227 task.exit 0 301 queuepush DetrendResidExposures -replace -key 1:2:5 "DONE $options:0" 302 end 303 304 # default exit status 305 task.exit default 306 showcommand failure 307 queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0" 308 $DetrendResidExpFail ++ 309 end 310 311 # operation times out? 312 task.exit timeout 313 showcommand timeout 314 queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0" 315 $DetrendResidExpFail ++ 316 end 317 end 228 book setword DetrendResidExposures $options:0 state DONE 229 end 230 231 # default exit status 232 task.exit default 233 showcommand failure 234 book setword DetrendResidExposures $options:0 state FAIL 235 end 236 237 # operation times out? 238 task.exit timeout 239 showcommand timeout 240 book setword DetrendResidExposures $options:0 state TIMEOUT 241 end 242 end -
branches/eam_branch_00/ippTasks/detrend.stack.pro
r10745 r11011 1 1 ## this file contains the tasks for running the detrend stacking stage 2 2 3 # get a list of imfiles to stack4 # dettool -tostack -simple5 # 1 0 bias mc6 # DET_ID ITER TYPE CLASS_ID7 8 # function to stack the imfiles:9 # detrend_stack.pl 1 0 mc bias10 11 ## these tasks use the queue DetrendStackClass12 ## the DetrendStackClass queue contains:13 ## STATE DET_ID ITER TYPE CLASS_ID CAMERA14 15 queueinit DetrendStackClass16 3 if ($?network == 0) 17 4 $network = 1 … … 20 7 $parallel = 0 21 8 end 22 23 $DetrendStackClassFail = 024 25 9 if ($?VERBOSE == 0) 26 10 echo "VERBOSE not defined: load pantasks.pro first" … … 31 15 break 32 16 end 17 33 18 $LOGSUBDIR = $LOGDIR/detrend 34 19 exec mkdir -p $LOGSUBDIR 20 21 book init DetrendStackClass 35 22 36 23 macro detstack.status … … 39 26 end 40 27 41 # select images ready for copy42 # new entries are added to queueDetrendStackClass28 # select images ready for detrend_stack.pl 29 # new entries are added to DetrendStackClass 43 30 # compare the new list with the ones already selected 44 31 task dettool.stack.load … … 53 40 54 41 task.exec 55 command dettool -tostacked - simple -limit {$DetrendStackClassFail + 20}42 command dettool -tostacked -limit 20 56 43 end 57 44 58 45 # success 59 46 task.exit 0 60 local i Nqueue 61 62 # compare output with newImage queue 63 # only add entries which do not exist in queue 64 queuesize stdout -var Nqueue 65 for i 0 $Nqueue 66 queuepop stdout -var line 67 if ($VERBOSE > 2) 68 echo $line 69 end 70 list word -split $line 71 $DET_ID = $word:0 72 $ITER = $word:1 73 $DET_TYPE = $word:2 74 $CLASS_ID = $word:3 75 $CAMERA = $word:4 76 queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $CLASS_ID $CAMERA" 47 # convert 'stdout' to book format 48 ipptool2book stdout DetrendStackClass -key det_id:iter:class_id -uniq 49 if ($VERBOSE > 2) 50 book listbook DetrendStackClass 77 51 end 78 52 79 53 # delete existing entries which are DONE 80 queuedrop DetrendStackClass -key 0DONE54 book delpage DetrendStackClass -key state DONE 81 55 end 82 56 … … 99 73 100 74 task.exec 101 queuesizeDetrendStackClass -var N75 book npages DetrendStackClass -var N 102 76 if ($N == 0) break 103 77 if ($network == 0) break 104 78 105 # look for new images on the internal DetrendStackClass queue 106 # caution with these 'if' statements: syntax errors 107 # will make the task fail without given a good status 108 queuepop DetrendStackClass -var line -key 0 NEW 109 if ("$line" == "NULL") break 79 # look for new images in DetrendStackClass 80 book getpage DetrendStackClass 0 -var pageName -key state NULL 81 if ($pageName == NULL) break 110 82 111 strpop line state 112 queuepush DetrendStackClass -replace -key 1:2:4 "RUN $line" 113 114 # the DetrendStackClass queue contains: 115 # STATE DET_ID ITER TYPE CLASS_ID 116 ## NOTE: the queue has the STATE prepended 117 # use local variables 118 list word -split $line 119 $DET_ID = $word:0 120 $ITER = $word:1 121 $DET_TYPE = $word:2 122 $CLASS_ID = $word:3 123 $CAMERA = $word:4 124 125 # XXX add $WORKDIR/$LOG_DIR 126 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log 127 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log 83 book setword DetrendStackClass $pageName state RUN 84 book getword DetrendStackClass $pageName det_id -var DET_ID 85 book getword DetrendStackClass $pageName iter -var ITER 86 book getword DetrendStackClass $pageName det_type -var DET_TYPE 87 book getword DetrendStackClass $pageName class_id -var CLASS_ID 88 book getword DetrendStackClass $pageName camera -var CAMERA 128 89 129 90 # specify choice of remote host: … … 134 95 end 135 96 136 # create example job options as a demonstration 137 options "$line" 97 # XXX add $WORKDIR/$LOG_DIR 98 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log 99 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log 100 101 # save the pageName for future reference below 102 options $pageName 103 104 # create the command line 138 105 if ($VERBOSE > 1) 139 106 echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA … … 144 111 # success 145 112 task.exit 0 146 queuepush DetrendStackClass -replace -key 1:2:4 "DONE $options:0"113 book setword DetrendStackClass $options:0 state DONE 147 114 end 148 115 … … 150 117 task.exit default 151 118 showcommand failure 152 queuepush DetrendStackClass -replace -key 1:2:4 "FAIL $options:0" 153 $DetrendStackClassFail ++ 119 book setword DetrendStackClass $options:0 state FAIL 154 120 end 155 121 … … 157 123 task.exit timeout 158 124 showcommand timeout 159 queuepush DetrendStackClass -replace -key 1:2:4 "TIMEOUT $options:0" 160 $DetrendStackClassFail ++ 125 book setword DetrendStackClass $options:0 state TIMEOUT 161 126 end 162 127 end -
branches/eam_branch_00/ippTasks/pantasks.pro
r10760 r11011 80 80 end 81 81 end 82 83 macro setcommand 84 local n 85 $command = "" 86 for n 1 $0 87 $command = $command $$n 88 end 89 if ($VERBOSE > 1) 90 echo "command $command" 91 end 92 command $command 93 end -
branches/eam_branch_00/ippTasks/phase0.pro
r10746 r11011 1 1 ## this file contains the tasks for running the phase 0 stage 2 3 ## these tasks use the queue Phase0Imfiles 4 ## the Phase0Imfiles queue contains: 5 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS 6 ## queue keys are counted from 0 7 8 queueinit Phase0Imfiles 9 queueinit Phase0Exposures 2 ## these tasks use the books Phase0Imfiles and Phase0Exposures 3 10 4 if ($?network == 0) 11 5 $network = 1 … … 14 8 $parallel = 0 15 9 end 16 17 $Phase0ImfileFail = 018 $Phase0ExpFail = 019 20 10 if ($?VERBOSE == 0) 21 11 echo "VERBOSE not defined: load pantasks.pro first" … … 26 16 break 27 17 end 18 28 19 $LOGSUBDIR = $LOGDIR/phase0 29 20 exec mkdir -p $LOGSUBDIR 30 21 22 book init Phase0Imfiles 23 book init Phase0Exposures 24 31 25 macro phase0.status 32 queueprint Phase0Imfiles 33 queueprint Phase0Exposures 34 echo "Phase0ImfileFail: $Phase0ImfileFail" 35 echo "Phase0ExpFail: $Phase0ExpFail" 26 book listbook Phase0Imfiles 27 book listbook Phase0Exposures 36 28 end 37 29 … … 50 42 51 43 task.exec 52 command p0tool -pendingimfile - simple -limit {$Phase0ImfileFail + 20}44 command p0tool -pendingimfile -limit 20 53 45 end 54 46 55 47 # success 56 48 task.exit 0 57 local i Nqueue 58 59 # compare output with Phase0Imfiles queue 60 # only add entries which don't exist in queue 61 queuesize stdout -var Nqueue 62 for i 0 $Nqueue 63 queuepop stdout -var line 64 if ($VERBOSE > 2) 65 echo $line 66 end 67 list word -split $line 68 $EXP_TAG = $word:0 69 $CLASS = $word:1 70 $CLASS_ID = $word:2 71 $URI = $word:3 72 queuepush Phase0Imfiles -uniq -key 1 "NEW $EXP_TAG $CLASS $CLASS_ID $URI" 49 # convert 'stdout' to book format 50 ipptool2book stdout Phase0Imfiles -key exp_tag -uniq 51 if ($VERBOSE > 2) 52 book listbook Phase0Imfiles 73 53 end 74 54 75 55 # delete existing entries which are DONE 76 queuedrop Phase0Imfiles -key 0DONE56 book delpage Phase0Imfiles -key state DONE 77 57 end 78 58 … … 88 68 end 89 69 90 91 92 # run the phase0imfile script on pending images 70 # run the phase0_imfile.pl script on pending images 93 71 task phase0.imfile.run 94 72 periods -poll $runpoll … … 97 75 98 76 task.exec 99 queuesizePhase0Imfiles -var N77 book npages Phase0Imfiles -var N 100 78 if ($N == 0) break 101 79 if ($network == 0) break 102 80 103 # look for new images on the internal DetrendImfilesToProcess queue 104 # caution with these 'if' statements: syntax errors 105 # will make the task fail without given a good status 106 queuepop Phase0Imfiles -var line -key 0 NEW 107 if ("$line" == "NULL") break 108 109 strpop line state 110 queuepush Phase0Imfiles -replace -key 1 "RUN $line" 111 112 ## the Phase0Imfiles queue contains (line has popped STATE) 113 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS 114 list word -split $line 115 $EXP_TAG = $word:0 116 $CLASS = $word:1 117 $CLASS_ID = $word:2 118 $URI = $word:3 119 120 # specify choice of remote host:(need to choose based on chips) 81 # look for new images in Phase0Imfiles 82 book getpage Phase0Imfiles 0 -var pageName -key state NULL 83 if ($pageName == NULL) break 84 85 book setword Phase0Imfiles $pageName state RUN 86 book getword Phase0Imfiles $pageName exp_tag -var EXP_TAG 87 book getword Phase0Imfiles $pageName class -var CLASS 88 book getword Phase0Imfiles $pageName class_id -var CLASS_ID 89 book getword Phase0Imfiles $pageName uri -var URI 90 91 # specify choice of remote host 121 92 if ($parallel) 122 93 host anyhost … … 133 104 exec mkdir -p $LOGSUBDIR/$base 134 105 106 # save the pageName for future reference below 107 options $pageName 108 135 109 # create the command line 136 options "$line"137 110 if ($VERBOSE > 1) 138 111 echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI … … 143 116 # success 144 117 task.exit 0 145 # phase0 imfile.pl updates DB tables, here we just update the queue146 queuepush Phase0Imfiles -replace -key 1 "DONE $options:0"118 # phase0_imfile.pl updates DB tables, here we just update the queue 119 book setword Phase0Imfiles $options:0 state DONE 147 120 end 148 121 … … 150 123 task.exit default 151 124 showcommand failure 152 queuepush Phase0Imfiles -replace -key 1 "FAIL $options:0" 153 $Phase0ImfileFail ++ 125 book setword Phase0Imfiles $options:0 state FAIL 154 126 end 155 127 … … 157 129 task.exit timeout 158 130 showcommand timeout 159 queuepush Phase0Imfiles -replace -key 1 "TIMEOUT $options:0" 160 $Phase0ImfileFail ++ 161 end 162 end 163 164 165 166 ## these tasks use the queue Phase0Exposures 167 ## the Phase0Exposures queue contains: 168 ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS 169 170 # select images ready for phase0 analysis 171 # new entries are added to queue Phase0Exposures 172 # compare the new list with the ones already selected 131 book setword Phase0Imfiles $options:0 state TIMEOUT 132 end 133 end 134 135 # select exposures ready for phase0_exp.pl 173 136 task phase0.exp.load 174 137 host local … … 182 145 183 146 task.exec 184 command p0tool -pendingexp - simple -limit {$Phase0ExpFail + 20}147 command p0tool -pendingexp -limit 20 185 148 end 186 149 187 150 # success 188 151 task.exit 0 189 local i Nqueue 190 191 # compare output with Phase0Exposures queue 192 # only add entries which don't exist in queue 193 queuesize stdout -var Nqueue 194 for i 0 $Nqueue 195 queuepop stdout -var line 196 if ($VERBOSE > 2) 197 echo $line 198 end 199 list word -split $line 200 $EXP_TAG = $word:0 201 # $EXP_ID = $word:1 202 # $CAMERA = $word:2 203 # $TELESCOPE = $word:3 204 # $DATEOBS = $word:4 205 # $TYPE = $word:5 206 # $IMFILES = $word:6 207 queuepush Phase0Exposures -uniq -key 1 "NEW $EXP_TAG" 152 # convert 'stdout' to book format 153 ipptool2book stdout Phase0Exposures -key exp_tag -uniq 154 if ($VERBOSE > 2) 155 book listbook Phase0Exposures 208 156 end 209 157 210 158 # delete existing entries which are DONE 211 queuedrop Phase0Exposures -key 0DONE159 book delpage Phase0Exposures -key state DONE 212 160 end 213 161 214 162 # default exit status 215 163 task.exit default 216 showcommand 217 end 218 219 # operation times out? 220 task.exit timeout 221 showcommand 222 end 223 end 224 225 226 227 # run the phase0imfile script on pending images 164 showcommand failure 165 end 166 167 # operation times out? 168 task.exit timeout 169 showcommand timeout 170 end 171 end 172 173 # run the phase0_exp.pl script on pending exposures 228 174 task phase0.exp.run 229 175 periods -poll $runpoll … … 232 178 233 179 task.exec 234 queuesizePhase0Exposures -var N180 book npages Phase0Exposures -var N 235 181 if ($N == 0) break 236 182 if ($network == 0) break 237 183 238 # look for new images on the internal DetrendImfilesToProcess queue239 # caution with these 'if' statements: syntax errors240 # will make the task fail without given a good status241 queuepop Phase0Exposures -var line -key 0 NEW 242 if ("$line" == "NULL") break243 244 strpop line state 245 queuepush Phase0Exposures -replace -key 1 "RUN $line"246 247 ## the Phase0Exposures queue contains:248 ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS249 list word -split $line250 $EXP_TAG = $word:0184 # look for new images in Phase0Exposures (state == NULL) 185 book getpage Phase0Exposures 0 -var pageName -key state NULL 186 if ($pageName == NULL) break 187 188 book setword Phase0Exposures $pageName state RUN 189 book getword Phase0Exposures $pageName exp_tag -var EXP_TAG 190 191 # specify choice of remote host:(need to choose based on chips) 192 if ($parallel) 193 host anyhost 194 else 195 host local 196 end 251 197 252 198 # XXX this is still a lame rule … … 258 204 exec mkdir -p $LOGSUBDIR/$base 259 205 260 # specify choice of remote host:(need to choose based on chips) 261 if ($parallel) 262 host anyhost 263 else 264 host local 265 end 206 # save the pageName for future reference below 207 options $pageName 266 208 267 209 # create the command line 268 options "$line"269 210 if ($VERBOSE > 1) 270 211 echo command phase0_exp.pl --exp_tag $EXP_TAG … … 276 217 task.exit 0 277 218 # phase0exp.pl updates DB tables, here we just update the queue 278 queuepush Phase0Exposures -replace -key 1 "DONE $options:0"219 book setword Phase0Exposures $options:0 state DONE 279 220 end 280 221 … … 282 223 task.exit default 283 224 showcommand 284 queuepush Phase0Exposures -replace -key 1 "FAIL $options:0" 285 $Phase0ExpFail 225 book setword Phase0Exposures $options:0 state FAIL 286 226 end 287 227 … … 289 229 task.exit timeout 290 230 showcommand 291 queuepush Phase0Exposures -replace -key 1 "TIMEOUT $options:0" 292 $Phase0ExpFail 231 book setword Phase0Exposures $options:0 state TIMEOUT 293 232 end 294 233 end -
branches/eam_branch_00/ippTasks/phase2.pro
r11008 r11011 1 1 ## this file contains the tasks for running the phase 0 stage 2 ## these tasks use the book p2PendingImfile 2 3 3 ## these tasks use the queue Phase2Imfiles4 ## the Phase2Imfiles queue contains:5 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS6 ## queue keys are counted from 07 8 # XXX change book names to match ippTool values?9 # XXX replace with book init p2pending10 queueinit Phase2Imfiles11 4 if ($?network == 0) 12 5 $network = 1 … … 15 8 $parallel = 0 16 9 end 17 18 $Phase2ImfileFail = 019 20 10 if ($?VERBOSE == 0) 21 11 echo "VERBOSE not defined: load pantasks.pro first" … … 26 16 break 27 17 end 18 28 19 $LOGSUBDIR = $LOGDIR/phase2 29 20 exec mkdir -p $LOGSUBDIR 30 21 22 book init p2PendingImfile 23 31 24 macro phase2.status 32 book listbook p2pending 33 echo "Phase2ImfileFail: $Phase2ImfileFail" 25 book listbook p2PendingImfile 34 26 end 35 27 36 28 # select images ready for phase2 analysis 37 # new entries are added to queue Phase2Imfiles38 # compare the new list with the ones already selected29 # new entries are added to p2PendingImfile 30 # skip already-present entries 39 31 task phase2.imfile.load 40 32 host local … … 48 40 49 41 task.exec 50 # command p2tool -pendingimfile -simple -limit {$Phase2ImfileFail + 20}51 # XXX make sure we remove FAIL and TIMEOUT entries52 42 command p2tool -pendingimfile -limit 20 53 43 end … … 55 45 # success 56 46 task.exit 0 57 # stdout from the task is stored on queue 'stdout'58 47 # convert 'stdout' to book format 59 # only add entries which don't exist in the book 60 # 'state' has value of NULL 61 ipptool2book stdout p2pending -key exp_tag:class_id -uniq 48 ipptool2book stdout p2PendingImfile -key exp_tag:class_id -uniq 62 49 if ($VERBOSE > 2) 63 book listbook p2 pending50 book listbook p2PendingImfile 64 51 end 65 52 66 53 # delete existing entries which are DONE 67 book delpage p2 pending-key state DONE54 book delpage p2PendingImfile -key state DONE 68 55 end 69 56 … … 79 66 end 80 67 81 # run the phase2 imfilescript on pending images68 # run the phase2.pl script on pending images 82 69 task phase2.imfile.run 83 70 periods -poll $runpoll … … 86 73 87 74 task.exec 88 book npages p2 pending-var N75 book npages p2PendingImfile -var N 89 76 if ($N == 0) break 90 77 if ($network == 0) break 91 78 92 # look for new images on the internal p2pending book 93 # caution with these 'if' statements: syntax errors 94 # will make the task fail without given a good status 95 book getpage p2pending 0 -var pageName -key state NULL 96 book setword p2pending $pageName state RUN 79 # look for new images in p2PendingImfile (state == NULL) 80 book getpage p2PendingImfile 0 -var pageName -key state NULL 81 if ($pageName == NULL) break 97 82 98 book getword p2pending $pageName camera -var CAMERA 99 book getword p2pending $pageName exp_tag -var EXP_TAG 100 book getword p2pending $pageName class_id -var CLASS_ID 101 book getword p2pending $pageName uri -var URI 83 book setword p2PendingImfile $pageName state RUN 84 book getword p2PendingImfile $pageName camera -var CAMERA 85 book getword p2PendingImfile $pageName exp_tag -var EXP_TAG 86 book getword p2PendingImfile $pageName class_id -var CLASS_ID 87 book getword p2PendingImfile $pageName uri -var URI 102 88 103 # specify choice of remote host:(need to choose based on chips) 89 # specify choice of remote host 90 # XXX need to choose based on chips 104 91 if ($parallel) 105 92 host anyhost … … 118 105 exec mkdir -p $LOGSUBDIR/$base 119 106 120 # save the p 2pending book pageName for future reference below107 # save the pageName for future reference below 121 108 options $pageName 122 109 … … 130 117 # success 131 118 task.exit 0 132 # phase2imfile.pl updates DB tables, here we just update the queue 133 book setword p2pending $options:0 state DONE 134 # queuepush Phase2Imfiles -replace -key 1:2 "DONE $options:0" 119 # phase2.pl updates DB tables, here we just update the queue 120 book setword p2PendingImfile $options:0 state DONE 135 121 end 136 122 … … 138 124 task.exit default 139 125 showcommand failure 140 book setword p2pending $options:0 state FAIL 141 # queuepush Phase2Imfiles -replace -key 1:2 "FAIL $options:0" 142 # drop failed entries from the book lists? 143 $Phase2ImfileFail ++ 126 book setword p2PendingImfile $options:0 state FAIL 144 127 end 145 128 146 # operation time sout?129 # operation timed out? 147 130 task.exit timeout 148 131 showcommand timeout 149 book setword p2pending $options:0 state TIMEOUT 150 # queuepush Phase2Imfiles -replace -key 1:2 "TIMEOUT $options:0" 151 $Phase2ImfileFail ++ 132 book setword p2PendingImfile $options:0 state TIMEOUT 152 133 end 153 134 end -
branches/eam_branch_00/ippTasks/phase3.pro
r10758 r11011 1 ## this file contains the tasks for running the phase 0 stage 1 ## this file contains the tasks for running the phase 3 stage 2 ## these tasks use the book Phase3Exposures 2 3 3 ## these tasks use the queue Phase3Exposures4 ## the Phase3Exposures queue contains:5 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS6 ## queue keys are counted from 07 8 queueinit Phase3Exposures9 4 if ($?network == 0) 10 5 $network = 1 … … 13 8 $parallel = 0 14 9 end 15 16 $Phase3ExposureFail = 017 18 10 if ($?VERBOSE == 0) 19 11 echo "VERBOSE not defined: load pantasks.pro first" … … 24 16 break 25 17 end 18 26 19 $LOGSUBDIR = $LOGDIR/phase3 27 20 exec mkdir -p $LOGSUBDIR 28 21 22 book init Phase3Exposures 23 29 24 macro phase3.status 30 queueprint Phase3Exposures 31 echo "Phase3ExposureFail: $Phase3ExposureFail" 25 book listbook Phase3Exposures 32 26 end 33 27 34 28 # select images ready for phase3 analysis 35 # new entries are added to queuePhase3Exposures36 # compare the new list with the ones already selected29 # new entries are added to Phase3Exposures 30 # skip already-present entries 37 31 task phase3.exp.load 38 32 host local … … 46 40 47 41 task.exec 48 command p3tool -pendingexp -simple42 command p3tool -pendingexp -limit 20 49 43 end 50 44 51 45 # success 52 46 task.exit 0 53 local i Nqueue 54 55 # compare output with Phase3Exposures queue 56 # only add entries which don't exist in queue 57 queuesize stdout -var Nqueue 58 for i 0 $Nqueue 59 queuepop stdout -var line 60 if ($VERBOSE > 2) 61 echo $line 62 end 63 list word -split $line 64 $EXP_TAG = $word:0 65 queuepush Phase3Exposures -uniq -key 1 "NEW $EXP_TAG" 47 # convert 'stdout' to book format 48 ipptool2book stdout Phase3Exposures -key exp_tag -uniq 49 if ($VERBOSE > 2) 50 book listbook Phase3Exposures 66 51 end 67 52 68 53 # delete existing entries which are DONE 69 queuedrop Phase3Exposures -key 0DONE54 book delpage Phase3Exposures -key state DONE 70 55 end 71 56 … … 88 73 89 74 task.exec 90 queuesizePhase3Exposures -var N75 book npages Phase3Exposures -var N 91 76 if ($N == 0) break 92 77 if ($network == 0) break 93 78 94 # look for new images on the internal DetrendExposuresToProcess queue 95 # caution with these 'if' statements: syntax errors 96 # will make the task fail without given a good status 97 queuepop Phase3Exposures -var line -key 0 NEW 98 if ("$line" == "NULL") break 79 # look for new images in Phase3Exposures (state == NULL) 80 book getpage Phase3Exposures 0 -var pageName -key state NULL 81 if ($pageName == NULL) break 99 82 100 strpop line state 101 queuepush Phase3Exposures -replace -key 1 "RUN $line" 83 book setword Phase3Exposures $pageName state RUN 84 book getword Phase3Exposures $pageName camera -var CAMERA 85 book getword Phase3Exposures $pageName exp_tag -var EXP_TAG 102 86 103 ## the Phase3Exposures queue contains: 104 ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS 105 list word -split $line 106 $EXP_TAG = $word:0 87 # specify choice of remote host:(need to choose based on chips) 88 if ($parallel) 89 host anyhost 90 else 91 host local 92 end 107 93 108 94 # XXX this is still a lame rule … … 114 100 exec mkdir -p $LOGSUBDIR/$base 115 101 116 # specify choice of remote host:(need to choose based on chips) 117 if ($parallel) 118 host anyhost 119 else 120 host local 121 end 102 # save the pageName for future reference below 103 options $pageName 122 104 123 105 # create the command line 124 options "$line"125 106 if ($VERBOSE > 1) 126 107 echo command phase3.pl --exp_tag $EXP_TAG 127 108 end 128 command phase3.pl --exp_tag $EXP_TAG --camera CTIO_MOSAIC2109 command phase3.pl --exp_tag $EXP_TAG --camera $CAMERA 129 110 end 130 111 … … 132 113 task.exit 0 133 114 # phase3exp.pl updates DB tables, here we just update the queue 134 queuepush Phase3Exposures -replace -key 1 "DONE $options:0"115 book setword Phase3Exposures $options:0 state DONE 135 116 end 136 117 … … 138 119 task.exit default 139 120 showcommand failure 140 queuepush Phase3Exposures -replace -key 1 "FAIL $options:0" 141 $Phase3ExposureFail ++ 121 book setword Phase3Exposures $options:0 state FAIL 142 122 end 143 123 … … 145 125 task.exit timeout 146 126 showcommand timeout 147 queuepush Phase3Exposures -replace -key 1 "TIMEOUT $options:0" 148 $Phase3ExposureFail ++ 127 book setword Phase3Exposures $options:0 state TIMEOUT 149 128 end 150 129 end
Note:
See TracChangeset
for help on using the changeset viewer.
