Changeset 11210
- Timestamp:
- Jan 19, 2007, 5:24:59 PM (19 years ago)
- Location:
- trunk/ippTasks
- Files:
-
- 11 edited
-
detrend.mkruns.pro (modified) (1 diff)
-
detrend.norm.pro (modified) (12 diffs)
-
detrend.process.pro (modified) (10 diffs)
-
detrend.reject.pro (modified) (7 diffs)
-
detrend.resid.pro (modified) (9 diffs)
-
detrend.stack.pro (modified) (7 diffs)
-
notes.txt (modified) (1 diff)
-
pantasks.pro (modified) (5 diffs)
-
phase0.pro (modified) (8 diffs)
-
phase2.pro (modified) (7 diffs)
-
phase3.pro (modified) (8 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippTasks/detrend.mkruns.pro
r9506 r11210 1 ## XXX this script needs to be updated to use books, not queues 2 ## XXX this script needs to be reconsidered... 1 3 2 4 ## This script defines tasks which regularly initiate the detrend -
trunk/ippTasks/detrend.norm.pro
r10745 r11210 1 2 ## this file contains the tasks for running the detrend normalization stage 3 4 ## these tasks use the queue DetrendNorm 5 ## the DetrendNorm queue contains: 6 ## STATE DET_ID ITER 7 8 queueinit DetrendNorm 9 queueinit DetrendNormStat 10 queueinit DetrendNormExp 1 ## this file contains the tasks for running the detrend normalization stages 2 ## these tasks use the books detPendingNormStatImfile detPendingNormImfile detPendingNormExp 3 11 4 if ($?network == 0) 12 5 $network = 1 … … 15 8 $parallel = 0 16 9 end 17 18 $DetrendNormFail = 019 $DetrendNormStatFail = 020 $DetrendNormExpFail = 021 22 10 if ($?VERBOSE == 0) 23 11 echo "VERBOSE not defined: load pantasks.pro first" … … 28 16 break 29 17 end 18 30 19 $LOGSUBDIR = $LOGDIR/detrend 31 20 exec mkdir -p $LOGSUBDIR 32 21 22 book init detPendingNormStatImfile 23 book init detPendingNormImfile 24 book init detPendingNormExp 25 33 26 macro detnorm.status 34 queueprint DetrendNorm 35 queueprint DetrendNormStat 36 queueprint DetrendNormExp 37 echo "DetrendNormFail: $DetrendNormFail" 38 echo "DetrendNormStatFail: $DetrendNormStatFail" 39 echo "DetrendNormExpFail: $DetrendNormExpFail" 27 book listbook detPendingNormStatImfile 28 book listbook detPendingNormImfile 29 book listbook detPendingNormExp 40 30 end 41 31 42 32 # select images ready for copy 43 # new entries are added to queue DetrendNormStat33 # new entries are added to detPendingNormStatImfile 44 34 # compare the new list with the ones already selected 45 35 task dettool.normstat.load … … 50 40 periods -timeout 30 51 41 52 stdout $LOGSUBDIR/dettool.normstat.log42 stdout NULL 53 43 stderr $LOGSUBDIR/dettool.normstat.log 54 44 55 45 task.exec 56 command dettool -tonormalizedstat -simple -limit {$DetrendNormStatFail + 20}46 command dettool -tonormalizedstat -limit 20 57 47 end 58 48 59 49 # success 60 50 task.exit 0 61 # XXX is it necessary for these to be local? 62 # drop after this task macro is done? 63 local i Nqueue 64 65 # compare output with newImage queue 66 # only add entries which don't exist in queue 67 queuesize stdout -var Nqueue 68 for i 0 $Nqueue 69 queuepop stdout -var line 70 if ($VERBOSE > 2) 71 echo $line 72 end 73 list word -split $line 74 $DET_ID = $word:0 75 $DET_TYPE = $word:1 76 $ITER = $word:2 77 $CAMERA = $word:3 78 queuepush DetrendNormStat -uniq -key 1:3 "NEW $DET_ID $DET_TYPE $ITER $CAMERA" 51 # convert 'stdout' to book format 52 ipptool2book stdout detPendingNormStatImfile -key det_id:iteration -uniq 53 if ($VERBOSE > 2) 54 book listbook detPendingNormStatImfile 79 55 end 80 56 81 57 # delete existing entries which are DONE 82 queuedrop DetrendNormStat -key 0 DONE 83 end 84 85 # locked list 58 book delpage detPendingNormStatImfile -key state DONE 59 end 60 86 61 # default exit status 87 62 task.exit default … … 102 77 103 78 task.exec 104 queuesize DetrendNormStat-var N79 book npages detPendingNormStatImfile -var N 105 80 if ($N == 0) break 106 81 if ($network == 0) break 107 82 108 # look for new images on the internal DetrendNormStat queue 109 # caution with these 'if' statements: syntax errors 110 # will make the task fail without given a good status 111 queuepop DetrendNormStat -var line -key 0 NEW 112 if ("$line" == "NULL") break 113 114 strpop line state 115 queuepush DetrendNormStat -replace -key 1:3 "RUN $line" 116 117 # the DetrendNormStat queue contains: 118 # STATE DET_ID ITER 119 list word -split $line 120 $DET_ID = $word:0 121 $DET_TYPE = $word:1 122 $ITER = $word:2 123 $CAMERA = $word:3 124 125 # XXX add $WORKDIR/$LOG_DIR 126 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 127 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 83 # look for new images in detPendingNormStatImfile 84 book getpage detPendingNormStatImfile 0 -var pageName -key state NULL 85 if ($pageName == NULL) break 86 87 book setword detPendingNormStatImfile $pageName state RUN 88 book getword detPendingNormStatImfile $pageName det_id -var DET_ID 89 book getword detPendingNormStatImfile $pageName det_type -var DET_TYPE 90 book getword detPendingNormStatImfile $pageName iteration -var ITERATION 91 book getword detPendingNormStatImfile $pageName camera -var CAMERA 128 92 129 93 # specify choice of remote host: … … 134 98 end 135 99 100 # XXX add $WORKDIR/$LOG_DIR 101 # XXX use ipp_filename.pl to lookup output file names 102 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 103 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATIION.log 104 105 # save the pageName for future reference below 106 options $pageName 107 136 108 # create example job options as a demonstration 137 options "$line"138 109 if ($VERBOSE > 1) 139 echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE 140 end 141 command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE 142 end 143 144 # success 145 task.exit 0 146 queuepush DetrendNormStat -replace -key 1:3 "DONE $options:0" 147 end 148 149 # default exit status 150 task.exit default 151 showcommand failure 152 queuepush DetrendNormStat -replace -key 1:3 "FAIL $options:0" 153 $DetrendNormStatFail ++ 154 end 155 156 # operation times out? 157 task.exit timeout 158 showcommand timeout 159 queuepush DetrendNormStat -replace -key 1:3 "TIMEOUT $options:0" 160 $DetrendNormStatFail ++ 110 echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE 111 end 112 command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE 113 end 114 115 # default exit status 116 task.exit default 117 process_exit detPendingNormStatImfile $options:0 $JOB_STATUS 118 end 119 120 # operation times out? 121 task.exit timeout 122 showcommand timeout 123 book setword detPendingNormStatImfile $options:0 state TIMEOUT 161 124 end 162 125 end 163 126 164 127 # select images ready for copy 165 # new entries are added to queue DetrendNorm128 # new entries are added to detPendingNormImfile 166 129 # compare the new list with the ones already selected 167 130 task dettool.norm.load … … 172 135 periods -timeout 30 173 136 174 stdout $LOGSUBDIR/dettool.norm.log137 stdout NULL 175 138 stderr $LOGSUBDIR/dettool.norm.log 176 139 177 140 task.exec 178 command dettool -tonormalize -simple -limit {$DetrendNormFail + 20}141 command dettool -tonormalize -limit 20 179 142 end 180 143 181 144 # success 182 145 task.exit 0 183 # XXX is it necessary for these to be local? 184 # drop after this task macro is done? 185 local i Nqueue 186 187 # compare output with newImage queue 188 # only add entries which don't exist in queue 189 queuesize stdout -var Nqueue 190 for i 0 $Nqueue 191 queuepop stdout -var line 192 if ($VERBOSE > 2) 193 echo $line 194 end 195 list word -split $line 196 $DET_TYPE = $word:0 197 $CAMERA = $word:1 198 $URI = $word:2 199 $DET_ID = $word:3 200 $ITER = $word:4 201 $CLASS_ID = $word:5 202 $NORM = $word:6 203 queuepush DetrendNorm -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM" 146 # convert 'stdout' to book format 147 ipptool2book stdout detPendingNormImfile -key det_id:iteration -uniq 148 if ($VERBOSE > 2) 149 book listbook detPendingNormImfile 204 150 end 205 151 206 152 # delete existing entries which are DONE 207 queuedrop DetrendNorm -key 0 DONE 208 end 209 210 # locked list 153 book delpage detPendingNormImfile -key state DONE 154 end 155 211 156 # default exit status 212 157 task.exit default … … 227 172 228 173 task.exec 229 queuesize DetrendNorm-var N174 book npages detPendingNormImfile -var N 230 175 if ($N == 0) break 231 176 if ($network == 0) break 232 177 233 # look for new images on the internal DetrendNorm queue 234 # caution with these 'if' statements: syntax errors 235 # will make the task fail without given a good status 236 queuepop DetrendNorm -var line -key 0 NEW 237 if ("$line" == "NULL") break 238 239 strpop line state 240 queuepush DetrendNorm -replace -key 4:5 "RUN $line" 241 242 # the DetrendNorm queue contains: 243 # STATE DET_ID ITER 244 list word -split $line 245 $DET_TYPE = $word:0 246 $CAMERA = $word:1 247 $URI = $word:2 248 $DET_ID = $word:3 249 $ITER = $word:4 250 $CLASS_ID = $word:5 251 $NORM = $word:6 252 253 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 254 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 178 # look for new images in detPendingNormImfile 179 book getpage detPendingNormImfile 0 -var pageName -key state NULL 180 if ($pageName == NULL) break 181 182 book setword detPendingNormImfile $pageName state RUN 183 book getword detPendingNormImfile $pageName det_type -var DET_TYPE 184 book getword detPendingNormImfile $pageName camera -var CAMERA 185 book getword detPendingNormImfile $pageName uri -var URI 186 book getword detPendingNormImfile $pageName det_id -var DET_ID 187 book getword detPendingNormImfile $pageName iteration -var ITERATION 188 book getword detPendingNormImfile $pageName class_id -var CLASS_ID 189 book getword detPendingNormImfile $pageName norm -var NORM 255 190 256 191 # specify choice of remote host: … … 261 196 end 262 197 198 # XXX use ipp_filename.pl to lookup output file names 199 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 200 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 201 202 # save the pageName for future reference below 203 options $pageName 204 263 205 # create example job options as a demonstration 264 options "$line"265 206 if ($VERBOSE > 1) 266 echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE 267 end 268 command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE 269 end 270 271 # success 272 task.exit 0 273 queuepush DetrendNorm -replace -key 4:5 "DONE $options:0" 274 end 275 276 # default exit status 277 task.exit default 278 showcommand failure 279 queuepush DetrendNorm -replace -key 4:5 "FAIL $options:0" 280 $DetrendNormFail ++ 281 end 282 283 # operation times out? 284 task.exit timeout 285 showcommand timeout 286 queuepush DetrendNorm -replace -key 4:5 "TIMEOUT $options:0" 287 $DetrendNormFail ++ 207 echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE 208 end 209 command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE 210 end 211 212 # default exit status 213 task.exit default 214 process_exit detPendingNormImfile $options:0 state FAIL 215 end 216 217 # operation times out? 218 task.exit timeout 219 showcommand timeout 220 book setword detPendingNormImfile $options:0 state TIMEOUT 288 221 end 289 222 end 290 223 291 224 # select images ready for copy 292 # new entries are added to queue DetrendNormExp225 # new entries are added to detPendingNormExp 293 226 # compare the new list with the ones already selected 294 227 task dettool.normexp.load … … 299 232 periods -timeout 30 300 233 301 stdout $LOGSUBDIR/dettool.normexp.log234 stdout NULL 302 235 stderr $LOGSUBDIR/dettool.normexp.log 303 236 304 237 task.exec 305 command dettool -tonormalizedexp -simple -limit {$DetrendNormExpFail + 20}238 command dettool -tonormalizedexp -limit 20 306 239 end 307 240 308 241 # success 309 242 task.exit 0 310 # XXX is it necessary for these to be local? 311 # drop after this task macro is done? 312 local i Nqueue 313 314 # compare output with newImage queue 315 # only add entries which don't exist in queue 316 queuesize stdout -var Nqueue 317 for i 0 $Nqueue 318 queuepop stdout -var line 319 if ($VERBOSE > 2) 320 echo $line 321 end 322 list word -split $line 323 $DET_ID = $word:0 324 $ITER = $word:1 325 $DET_TYPE = $word:2 326 $CAMERA = $word:3 327 #$TELESCOPE= $word:4 328 #$EXP_TYPE = $word:5 329 #$IMFILES = $word:6 330 queuepush DetrendNormExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA" 243 # convert 'stdout' to book format 244 ipptool2book stdout detPendingNormExp -key det_id:iteration -uniq 245 if ($VERBOSE > 2) 246 book listbook detPendingNormExp 331 247 end 332 248 333 249 # delete existing entries which are DONE 334 queuedrop DetrendNormExp -key 0 DONE 335 end 336 337 # locked list 250 book delpage detPendingNormExp -key state DONE 251 end 252 338 253 # default exit status 339 254 task.exit default … … 354 269 355 270 task.exec 356 queuesize DetrendNormExp -var N 357 271 book npages detPendingNormExp -var N 358 272 if ($N == 0) break 359 273 if ($network == 0) break 360 274 361 # look for new images on the internal DetrendNormExp queue 362 # caution with these 'if' statements: syntax errors 363 # will make the task fail without given a good status 364 queuepop DetrendNormExp -var line -key 0 NEW 365 if ("$line" == "NULL") break 366 367 strpop line state 368 queuepush DetrendNormExp -replace -key 1:2 "RUN $line" 369 370 # the DetrendNormExp queue contains: 371 # STATE DET_ID ITER 372 list word -split $line 373 $DET_ID = $word:0 374 $ITER = $word:1 375 $DET_TYPE = $word:2 376 $CAMERA = $word:3 377 #$TELESCOPE= $word:4 378 #$EXP_TYPE = $word:5 379 #$IMFILES = $word:6 380 381 # XXX add $WORKDIR/$LOG_DIR 382 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 383 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 275 # look for new images in detPendingNormExp 276 book getpage detPendingNormExp 0 -var pageName -key state NULL 277 if ($pageName == NULL) break 278 279 book setword detPendingNormExp $pageName state RUN 280 book getword detPendingNormExp $pageName det_id -var DET_ID 281 book getword detPendingNormExp $pageName iteration -var ITERATION 282 book getword detPendingNormExp $pageName det_type -var DET_TYPE 283 book getword detPendingNormExp $pageName camera -var CAMERA 384 284 385 285 # specify choice of remote host: … … 390 290 end 391 291 292 # XXX add $WORKDIR/$LOG_DIR 293 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 294 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 295 296 # save the pageName for future reference below 297 options $pageName 298 392 299 # create example job options as a demonstration 393 options "$line"394 300 if ($VERBOSE > 1) 395 echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE 396 end 397 command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE 398 end 399 400 # success 401 task.exit 0 402 queuepush DetrendNormExp -replace -key 1:2 "DONE $options:0" 403 end 404 405 # default exit status 406 task.exit default 407 showcommand failure 408 queuepush DetrendNormExp -replace -key 1:2 "FAIL $options:0" 409 $DetrendNormExpFail ++ 410 end 411 412 # operation times out? 413 task.exit timeout 414 showcommand timeout 415 queuepush DetrendNormExp -replace -key 1:2 "TIMEOUT $options:0" 416 $DetrendNormExpFail ++ 417 end 418 end 301 echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE 302 end 303 command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE 304 end 305 306 # default exit status 307 task.exit default 308 process_exit detPendingNormExp $options:0 $JOB_STATUS 309 end 310 311 # operation times out? 312 task.exit timeout 313 showcommand timeout 314 book setword detPendingNormExp $options:0 state TIMEOUT 315 end 316 end -
trunk/ippTasks/detrend.process.pro
r10745 r11210 1 1 ## this file contains the tasks for running the detrend processing stage 2 3 # query for pending imfiles to process: 4 # dettool -raw -simple 5 # 1 bias 850131b fpa mc 850131b.fits BIAS r.MP9601 1.656000 5.496118 0.349071 0.000000 1155.934692 107.037283 46.214947 37.160000 105.480000 -133.067001 0.000000 6 # this returns many columns for manual operations. we strip all but the first few: 7 8 # process a single imfile 9 # detrend_process.pl 1 850131b mc bias 850131b.fits 850131b 10 11 ## these tasks use the queue DetrendProcessImfiles 12 ## the DetrendProcessImfiles queue contains: 13 ## STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI 14 15 queueinit DetrendProcessImfiles 16 queueinit DetrendProcessExposures 2 ## these tasks use the books detPendingProcessedImfile and detPendingProcessedExp 3 17 4 if ($?network == 0) 18 5 $network = 1 … … 21 8 $parallel = 0 22 9 end 23 24 $DetrendProcessImfileFail = 025 $DetrendProcessExpFail = 026 27 10 if ($?VERBOSE == 0) 28 11 echo "VERBOSE not defined: load pantasks.pro first" … … 33 16 break 34 17 end 18 35 19 $LOGSUBDIR = $LOGDIR/detrend 36 20 exec mkdir -p $LOGSUBDIR 37 21 22 book init detPendingProcessedImfile 23 book init detPendingProcessedExp 24 38 25 macro detproc.status 39 queueprint DetrendProcessImfiles 40 queueprint DetrendProcessExposures 41 echo "DetrendProcessImfileFail: $DetrendProcessImfileFail" 42 echo "DetrendProcessExpFail: $DetrendProcessExpFail" 26 book listbook detPendingProcessedImfile 27 book listbook detPendingProcessedExp 43 28 end 44 29 45 30 # select images ready for copy 46 # new entries are added to queue DetrendProcessImfiles31 # new entries are added to detPendingProcessedImfile 47 32 # compare the new list with the ones already selected 48 33 task dettool.process.load … … 53 38 periods -timeout 30 54 39 55 stdout $LOGSUBDIR/detproc.imfile.log40 stdout NULL 56 41 stderr $LOGSUBDIR/detproc.imfile.log 57 42 58 43 task.exec 59 command dettool -toprocessedimfile - simple -limit {$DetrendProcessImfileFail + 20}44 command dettool -toprocessedimfile -limit 20 60 45 end 61 46 62 47 # success 63 48 task.exit 0 64 # XXX is it necessary for these to be local? 65 # drop after this task macro is done? 66 local i Nqueue 67 68 # compare output with newImage queue 69 # only add entries which don't exist in queue 70 queuesize stdout -var Nqueue 71 for i 0 $Nqueue 72 queuepop stdout -var line 73 if ($VERBOSE > 2) 74 echo $line 75 end 76 list word -split $line 77 $DET_ID = $word:0 78 $DET_TYPE = $word:1 79 $EXP_TAG = $word:2 80 $CLASS = $word:3 81 $CLASS_ID = $word:4 82 $URI = $word:5 83 $CAMERA = $word:21 84 queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $DET_TYPE $EXP_TAG $CLASS $CLASS_ID $URI $CAMERA" 49 # convert 'stdout' to book format 50 ipptool2book stdout detPendingProcessedImfile -key det_id:exp_tag:class_id -uniq 51 if ($VERBOSE > 2) 52 book listbook detPendingProcessedImfile 85 53 end 86 54 87 55 # delete existing entries which are DONE 88 queuedrop DetrendProcessImfiles -key 0DONE56 book delpage detPendingProcessedImfile -key state DONE 89 57 end 90 58 … … 100 68 end 101 69 102 # copy new images, sending job to desired host70 # run detrend_process_imfile.ps on pending images 103 71 task dettool.process.run 104 72 periods -poll $runpoll … … 107 75 108 76 task.exec 109 queuesize DetrendProcessImfiles-var N77 book npages detPendingProcessedImfile -var N 110 78 if ($N == 0) break 111 79 if ($network == 0) break 112 80 113 # look for new images on the internal DetrendProcessImfiles queue 114 # caution with these 'if' statements: syntax errors 115 # will make the task fail without given a good status 116 queuepop DetrendProcessImfiles -var line -key 0 NEW 117 if ("$line" == "NULL") break 118 119 strpop line state 120 queuepush DetrendProcessImfiles -replace -key 1:3:5 "RUN $line" 121 122 ## the DetrendProcessImfiles queue contains: 123 # STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI 124 ## NOTE: the queue has the STATE prepended 125 list word -split $line 126 $DET_ID = $word:0 127 $DET_TYPE = $word:1 128 $EXP_TAG = $word:2 129 $CLASS = $word:3 130 $CLASS_ID = $word:4 131 $URI = $word:5 132 $CAMERA = $word:6 133 81 # look for new images in detPendingProcessedImfile 82 book getpage detPendingProcessedImfile 0 -var pageName -key state NULL 83 if ($pageName == NULL) break 84 85 book setword detPendingProcessedImfile $pageName state RUN 86 book getword detPendingProcessedImfile $pageName det_id -var DET_ID 87 book getword detPendingProcessedImfile $pageName det_type -var DET_TYPE 88 book getword detPendingProcessedImfile $pageName exp_tag -var EXP_TAG 89 book getword detPendingProcessedImfile $pageName class -var CLASS 90 book getword detPendingProcessedImfile $pageName class_id -var CLASS_ID 91 book getword detPendingProcessedImfile $pageName uri -var URI 92 book getword detPendingProcessedImfile $pageName camera -var CAMERA 93 94 # specify choice of remote host: 95 if ($parallel) 96 host anyhost 97 else 98 host local 99 end 100 101 ## generate output log based on filerule 102 ## XXX use OUTPATH from db to set output path 103 ## XXX use ipp_filename.pl DETPROC.LOG $outroot $CLASS_ID` 134 104 $word = `basename $EXP_TAG | tr '.' ' '` 135 105 list word -split $word … … 139 109 exec mkdir -p $LOGSUBDIR/$base 140 110 111 # save the pageName for future reference below 112 options $pageName 113 114 # create example job options as a demonstration 115 if ($VERBOSE > 1) 116 echo command detrend_process_imfile.pl --det_id $DET_ID --det_type $DET_TYPE --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --input_uri $URI --camera $CAMERA 117 end 118 command detrend_process_imfile.pl --det_id $DET_ID --det_type $DET_TYPE --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --input_uri $URI --camera $CAMERA 119 end 120 121 # default exit status 122 task.exit default 123 process_exit detPendingProcessedImfile $options:0 $JOB_STATUS 124 end 125 126 # operation times out? 127 task.exit timeout 128 showcommand timeout 129 book setword detPendingProcessedImfile $options:0 state TIMEOUT 130 end 131 end 132 133 # select images ready for copy 134 # new entries are added to detPendingProcessedExp 135 # compare the new list with the ones already selected 136 task dettool.processexp.load 137 host local 138 139 periods -poll $loadpoll 140 periods -exec $loadexec 141 periods -timeout 30 142 143 stdout NULL 144 stderr $LOGSUBDIR/detproc.exp.log 145 146 task.exec 147 command dettool -toprocessedexp -limit 20 148 end 149 150 # success 151 task.exit 0 152 # convert 'stdout' to book format 153 ipptool2book stdout detPendingProcessedExp -key det_id:iteration:exp_tag -uniq 154 if ($VERBOSE > 2) 155 book listbook detPendingProcessedExp 156 end 157 158 # delete existing entries which are DONE 159 book delpage detPendingProcessedExp -key state DONE 160 end 161 162 # error 163 task.exit default 164 showcommand failure 165 end 166 167 # operation times out? 168 task.exit timeout 169 showcommand timeout 170 end 171 end 172 173 # copy new images, sending job to desired host 174 task dettool.processexp.run 175 periods -poll $runpoll 176 periods -exec $runexec 177 periods -timeout 30 178 179 task.exec 180 book npages detPendingProcessedExp -var N 181 if ($N == 0) break 182 if ($network == 0) break 183 184 # look for new exposures in detPendingProcessedExp 185 book getpage detPendingProcessedExp 0 -var pageName -key state NULL 186 if ($pageName == NULL) break 187 188 book setword detPendingProcessedExp $pageName state RUN 189 book getword detPendingProcessedExp $pageName det_id -var DET_ID 190 book getword detPendingProcessedExp $pageName iteration -var ITERATION 191 book getword detPendingProcessedExp $pageName det_type -var DET_TYPE 192 book getword detPendingProcessedExp $pageName exp_tag -var EXP_TAG 193 book getword detPendingProcessedExp $pageName camera -var CAMERA 194 141 195 # specify choice of remote host: 142 196 if ($parallel) … … 146 200 end 147 201 148 # create example job options as a demonstration149 options "$line"150 if ($VERBOSE > 1)151 echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA152 end153 command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA154 end155 156 # success157 task.exit 0158 queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0"159 # detrend_process_imfile.pl --force_exist --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA160 end161 162 # default exit status163 task.exit default164 showcommand failure165 queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0"166 $DetrendProcessImfileFail ++167 end168 169 # operation times out?170 task.exit timeout171 showcommand timeout172 queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0"173 $DetrendProcessImfileFail ++174 end175 end176 177 # select images ready for copy178 # new entries are added to queue DetrendProcessExposures179 # compare the new list with the ones already selected180 task dettool.processexp.load181 host local182 183 periods -poll $loadpoll184 periods -exec $loadexec185 periods -timeout 30186 187 stdout $LOGSUBDIR/detproc.exp.log188 stderr $LOGSUBDIR/detproc.exp.log189 190 task.exec191 command dettool -toprocessedexp -simple -limit {$DetrendProcessExpFail + 20}192 end193 194 # success195 task.exit 0196 # XXX is it necessary for these to be local?197 # drop after this task macro is done?198 local i Nqueue199 200 # compare output with newImage queue201 # only add entries which don't exist in queue202 queuesize stdout -var Nqueue203 for i 0 $Nqueue204 queuepop stdout -var line205 if ($VERBOSE > 2)206 echo $line207 end208 list word -split $line209 $DET_ID = $word:0210 $ITER = $word:1211 $DET_TYPE = $word:2212 $EXP_TAG = $word:3213 $CAMERA = $word:4214 queuepush DetrendProcessExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_TAG $CAMERA"215 end216 217 # delete existing entries which are DONE218 queuedrop DetrendProcessExposures -key 0 DONE219 end220 221 # error222 task.exit default223 showcommand failure224 end225 226 # operation times out?227 task.exit timeout228 showcommand timeout229 end230 end231 232 # copy new images, sending job to desired host233 task dettool.processexp.run234 periods -poll $runpoll235 periods -exec $runexec236 periods -timeout 30237 238 task.exec239 queuesize DetrendProcessExposures -var N240 if ($N == 0) break241 if ($network == 0) break242 243 # look for new images on the internal DetrendProcessExposures queue244 # caution with these 'if' statements: syntax errors245 # will make the task fail without given a good status246 queuepop DetrendProcessExposures -var line -key 0 NEW247 if ("$line" == "NULL") break248 249 strpop line state250 queuepush DetrendProcessExposures -replace -key 1:2:4 "RUN $line"251 252 list word -split $line253 $DET_ID = $word:0254 $ITER = $word:1255 $DET_TYPE = $word:2256 $EXP_TAG = $word:3257 $CAMERA = $word:4258 259 202 # XXX this is still a lame rule 260 203 $word = `basename $EXP_TAG | tr '.' ' '` 261 204 list word -split $word 262 205 $base = $word:0 263 264 206 stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log 265 207 stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log 266 208 exec mkdir -p $LOGSUBDIR/$base 267 209 268 # specify choice of remote host: 269 if ($parallel) 270 host anyhost 271 else 272 host local 273 end 210 # save the pageName for future reference below 211 options $pageName 274 212 275 213 # create example job options as a demonstration 276 options "$line"277 214 if ($VERBOSE > 1) 278 215 echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA … … 282 219 283 220 # success 284 task.exit 0 285 queuepush DetrendProcessExposures -replace -key 1:2:4 "DONE $options:0" 286 end 287 288 # default exit status 289 task.exit default 290 showcommand failure 291 queuepush DetrendProcessExposures -replace -key 1:2:4 "FAIL $options:0" 292 $DetrendProcessExpFail ++ 221 task.exit default 222 process_exit detPendingProcessedExp $options:0 $JOB_STATUS 293 223 end 294 224 … … 296 226 task.exit timeout 297 227 showcommand timeout 298 queuepush DetrendProcessExposures -replace -key 1:2:4 "TIMEOUT $options:0" 299 $DetrendProcessExpFail ++ 300 end 301 end 228 book setword detPendingProcessedExp $options:0 state TIMEOUT 229 end 230 end -
trunk/ippTasks/detrend.reject.pro
r10745 r11210 1 ## this file contains the tasks for running the detrend processing stage 2 ## these tasks use the book detRejectExp 1 3 2 ## this file contains the tasks for running the detrend processing stage3 4 # dettool -residdetrun -simple5 # 1 0 bias6 # DET_ID ITER TYPE7 8 # detrend_reject_exp.pl 1 0 bias9 # detrend_reject_exp.pl DET_ID ITER TYPE10 11 ## these tasks use the queue DetrendResidExp12 ## the DetrendRejectExp queue contains:13 ## STATE DET_ID ITER TYPE14 15 queueinit DetrendRejectExp16 4 if ($?network == 0) 17 5 $network = 1 … … 20 8 $parallel = 0 21 9 end 22 23 $DetrendRejectExpFail = 024 25 10 if ($?VERBOSE == 0) 26 11 echo "VERBOSE not defined: load pantasks.pro first" … … 31 16 break 32 17 end 18 33 19 $LOGSUBDIR = $LOGDIR/detrend 34 20 exec mkdir -p $LOGSUBDIR 35 21 22 book init detRejectExp 23 36 24 macro detreject.status 37 queueprint DetrendRejectExp 38 echo "DetrendRejectExpFail: $DetrendRejectExpFail" 25 book listbook detRejectExp 39 26 end 40 27 41 28 # select images ready for copy 42 # new entries are added to queue DetrendRejectExp29 # new entries are added to detRejectExp 43 30 # compare the new list with the ones already selected 44 31 task dettool.reject.load … … 49 36 periods -timeout 30 50 37 51 stdout $LOGSUBDIR/dettool.reject.log38 stdout NULL 52 39 stderr $LOGSUBDIR/dettool.reject.log 53 40 54 41 task.exec 55 command dettool -residdetrun - simple -limit {$DetrendRejectExpFail + 20}42 command dettool -residdetrun -limit 20 56 43 end 57 44 58 45 # success 59 46 task.exit 0 60 local i Nqueue 61 62 # compare output with newImage queue 63 # only add entries which don't exist in queue 64 queuesize stdout -var Nqueue 65 for i 0 $Nqueue 66 queuepop stdout -var line 67 if ($VERBOSE > 2) 68 echo $line 69 end 70 list word -split $line 71 $DET_ID = $word:0 72 $ITER = $word:1 73 $DET_TYPE = $word:2 74 $MODE = $word:3 75 $CAMERA = $word:4 76 queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $MODE $CAMERA" 47 # convert 'stdout' to book format 48 ipptool2book stdout detRejectExp -key det_id:iteration -uniq 49 if ($VERBOSE > 2) 50 book listbook detRejectExp 77 51 end 78 52 79 53 # delete existing entries which are DONE 80 queuedrop DetrendRejectExp -key 0DONE54 book delpage detRejectExp -key state DONE 81 55 end 82 56 … … 99 73 100 74 task.exec 101 queuesize DetrendRejectExp -var N75 book npages detRejectExp -var N 102 76 if ($N == 0) break 103 77 if ($network == 0) break 104 78 105 # look for new images on the internal DetrendRejectExp queue 106 # caution with these 'if' statements: syntax errors 107 # will make the task fail without given a good status 108 queuepop DetrendRejectExp -var line -key 0 NEW 109 if ("$line" == "NULL") break 79 # look for new images in detRejectExp 80 book getpage detRejectExp 0 -var pageName -key state NULL 81 if ($pageName == NULL) break 110 82 111 strpop line state 112 queuepush DetrendRejectExp -replace -key 1:2 "RUN $line" 113 114 ## the DetrendRejectExp queue contains: 115 ## STATE DET_ID ITER TYPE 116 ## NOTE: the queue has the STATE prepended 117 list word -split $line 118 $DET_ID = $word:0 119 $ITER = $word:1 120 $DET_TYPE = $word:2 121 $MODE = $word:3 122 $CAMERA = $word:4 123 124 # I'd like to add CAMERA to the log file... 125 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 126 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 83 book setword detRejectExp $pageName state RUN 84 book getword detRejectExp $pageName det_id -var DET_ID 85 book getword detRejectExp $pageName iteration -var ITERATION 86 book getword detRejectExp $pageName det_type -var DET_TYPE 87 book getword detRejectExp $pageName mode -var MODE 88 book getword detRejectExp $pageName camera -var CAMERA 127 89 128 90 # specify choice of remote host: … … 133 95 end 134 96 97 # I'd like to add CAMERA to the log file... 98 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 99 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 100 101 # save the pageName for future reference below 102 options $pageName 103 135 104 # create example job options as a demonstration 136 options "$line"137 105 if ($VERBOSE > 1) 138 echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA106 echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA 139 107 end 140 command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA108 command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA 141 109 end 142 143 # success144 task.exit 0145 queuepush DetrendRejectExp -replace -key 1:2 "DONE $options:0"146 end147 110 148 111 # default exit status 149 112 task.exit default 150 showcommand failure 151 queuepush DetrendRejectExp -replace -key 1:2 "FAIL $options:0" 152 $DetrendRejectExpFail ++ 113 process_exit detRejectExp $options:0 state FAIL 153 114 end 154 115 … … 156 117 task.exit timeout 157 118 showcommand failure 158 queuepush DetrendRejectExp -replace -key 1:2 "TIMEOUT $options:0" 159 $DetrendRejectExpFail ++ 119 book setword detRejectExp $options:0 state TIMEOUT 160 120 end 161 121 end -
trunk/ippTasks/detrend.resid.pro
r10745 r11210 1 2 1 ## this file contains the tasks for running the detrend processing stage 3 4 # dettool -toresidimfile -simple 5 # 0 bias 1 850131b mc o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582 6 # ITER TYPE DET_ID EXP_TAG CLASS_ID URI-IN 7 8 # detrend_create_resid.pl 1 0 850131b mc bias bias_mc_1_0.fit o_850131b.mc.fit 850131b 9 # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG) 10 # URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit 11 12 ## these tasks use the queue DetrendResidImfiles 13 ## the DetrendResidImfiles queue contains: 14 ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN 15 16 queueinit DetrendResidImfiles 17 queueinit DetrendResidExposures 2 ## these tasks use the books detPendingResidImfile and detPendingResidExp 3 18 4 if ($?network == 0) 19 5 $network = 1 … … 22 8 $parallel = 0 23 9 end 24 25 $DetrendResidImfileFail = 026 $DetrendResidExpFail = 027 28 10 if ($?VERBOSE == 0) 29 11 echo "VERBOSE not defined: load pantasks.pro first" … … 34 16 break 35 17 end 18 36 19 $LOGSUBDIR = $LOGDIR/detrend 37 20 exec mkdir -p $LOGSUBDIR 38 21 22 book init detPendingResidImfile 23 book init detPendingResidExp 24 39 25 macro detresid.status 40 queueprint DetrendResidImfiles 41 queueprint DetrendResidExposures 42 echo "DetrendResidImfileFail: $DetrendResidImfileFail" 43 echo "DetrendResidExpFail: $DetrendResidExpFail" 26 book listbook detPendingResidImfile 27 book listbook detPendingResidExp 44 28 end 45 29 46 30 # select images ready for copy 47 # new entries are added to queue DetrendResidImfiles31 # new entries are added to detPendingResidImfile 48 32 # compare the new list with the ones already selected 49 33 task dettool.resid.load … … 54 38 periods -timeout 30 55 39 56 stdout $LOGSUBDIR/detresid.log40 stdout NULL 57 41 stderr $LOGSUBDIR/detresid.log 58 42 59 43 task.exec 60 command dettool -toresidimfile - simple -limit {$DetrendResidImfileFail + 20}44 command dettool -toresidimfile -limit 20 61 45 end 62 46 63 47 # success 64 48 task.exit 0 65 # XXX is it necessary for these to be local? 66 # drop after this task macro is done? 67 local i Nqueue 68 69 # compare output with newImage queue 70 # only add entries which don't exist in queue 71 queuesize stdout -var Nqueue 72 for i 0 $Nqueue 73 queuepop stdout -var line 74 if ($VERBOSE > 2) 75 echo $line 76 end 77 list word -split $line 78 $DET_ID = $word:0 79 $ITER = $word:1 80 $DET_TYPE = $word:2 81 $MODE = $word:3 82 $EXP_TAG = $word:4 83 $CLASS_ID = $word:5 84 $URI = $word:6 85 $DET_URI = $word:7 86 $CAMERA = $word:8 87 ## must include the detID and iter in the key (if we have two running at the same time...) 88 ## Also needed to be in the output file (or path) 89 queuepush DetrendResidImfiles -uniq -key 1:2:5:6 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $CLASS_ID $URI $DET_URI $CAMERA" 49 # convert 'stdout' to book format 50 ipptool2book stdout detPendingResidImfile -key det_id:iteration:exp_tag:class_id -uniq 51 if ($VERBOSE > 2) 52 book listbook detPendingResidImfile 90 53 end 91 54 92 55 # delete existing entries which are DONE 93 queuedrop DetrendResidImfiles -key 0 DONE 94 end 95 96 # locked list 56 book delpage detPendingResidImfile -key state DONE 57 end 58 97 59 # default exit status 98 60 task.exit default … … 113 75 114 76 task.exec 115 queuesize DetrendResidImfiles-var N77 book npages detPendingResidImfile -var N 116 78 if ($N == 0) break 117 79 if ($network == 0) break 118 80 119 # look for new images on the internal DetrendResidImfiles queue 120 # caution with these 'if' statements: syntax errors 121 # will make the task fail without given a good status 122 queuepop DetrendResidImfiles -var line -key 0 NEW 123 if ("$line" == "NULL") break 124 125 strpop line state 126 queuepush DetrendResidImfiles -replace -key 1:2:5:6 "RUN $line" 127 128 ## the DetrendResidImfiles queue contains: 129 ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN 130 ## NOTE: the queue has the STATE prepended 131 list word -split $line 132 $DET_ID = $word:0 133 $ITER = $word:1 134 $DET_TYPE = $word:2 135 $MODE = $word:3 136 $EXP_TAG = $word:4 137 $CLASS_ID = $word:5 138 $URI = $word:6 139 $DET_URI = $word:7 140 $CAMERA = $word:8 141 142 $word = `basename $EXP_TAG | tr '.' ' '` 143 list word -split $word 144 $base = $word:0 145 stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log 146 stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log 147 exec mkdir -p $LOGSUBDIR/$base 81 # look for new images in detPendingResidImfile 82 book getpage detPendingResidImfile 0 -var pageName -key state NULL 83 if ($pageName == NULL) break 84 85 book setword detPendingResidImfile $pageName state RUN 86 book getword detPendingResidImfile $pageName det_id -var DET_ID 87 book getword detPendingResidImfile $pageName iteration -var ITERATION 88 book getword detPendingResidImfile $pageName det_type -var DET_TYPE 89 book getword detPendingResidImfile $pageName mode -var MODE 90 book getword detPendingResidImfile $pageName exp_tag -var EXP_TAG 91 book getword detPendingResidImfile $pageName class_id -var CLASS_ID 92 book getword detPendingResidImfile $pageName uri -var URI 93 book getword detPendingResidImfile $pageName det_uri -var DET_URI 94 book getword detPendingResidImfile $pageName camera -var CAMERA 148 95 149 96 # specify choice of remote host: … … 154 101 end 155 102 156 # create example job options as a demonstration 157 options "$line" 158 # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???) 159 # need to parse the subdir path for the detrend URL 160 # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits 161 echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE 162 command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE 163 end 164 165 # success 166 task.exit 0 167 queuepush DetrendResidImfiles -replace -key 1:2:5:6 "DONE $options:0" 168 end 169 170 # default exit status 171 task.exit default 172 showcommand failure 173 queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0" 174 $DetrendResidImfileFail ++ 175 end 176 177 # operation times out? 178 task.exit timeout 179 showcommand timeout 180 queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0" 181 $DetrendResidImfileFail ++ 103 # XXX use ipp_filename.pl to lookup output file names 104 $word = `basename $EXP_TAG | tr '.' ' '` 105 list word -split $word 106 $base = $word:0 107 stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.$CLASS_ID.log 108 stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.$CLASS_ID.log 109 exec mkdir -p $LOGSUBDIR/$base 110 111 # save the pageName for future reference below 112 options $pageName 113 114 # create command 115 if ($VERBOSE > 1) 116 echo command detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE 117 end 118 command detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE 119 end 120 121 # default exit status 122 task.exit default 123 process_exit detPendingResidImfile $options:0 $JOB_STATUS 124 end 125 126 # operation times out? 127 task.exit timeout 128 showcommand timeout 129 book setword detPendingResidImfile $options:0 state TIMEOUT 182 130 end 183 131 end 184 132 185 133 # select images ready for copy 186 # new entries are added to queue DetrendResidExposures134 # new entries are added to detPendingResidExp 187 135 # compare the new list with the ones already selected 188 136 task dettool.residexp.load … … 193 141 periods -timeout 30 194 142 195 stdout $LOGSUBDIR/detresid.exp.log143 stdout NULL 196 144 stderr $LOGSUBDIR/detresid.exp.log 197 145 198 146 task.exec 199 command dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20}147 command dettool -toresidexp -limit 20 200 148 end 201 149 202 150 # success 203 151 task.exit 0 204 # XXX is it necessary for these to be local? 205 # drop after this task macro is done? 206 local i Nqueue 207 208 # compare output with newImage queue 209 # only add entries which don't exist in queue 210 queuesize stdout -var Nqueue 211 for i 0 $Nqueue 212 queuepop stdout -var line 213 if ($VERBOSE > 2) 214 echo $line 215 end 216 list word -split $line 217 $DET_ID = $word:0 218 $ITER = $word:1 219 $DET_TYPE = $word:2 220 $MODE = $word:3 221 $EXP_TAG = $word:4 222 $INCLUDE = $word:5 223 $CAMERA = $word:6 224 queuepush DetrendResidExposures -uniq -key 1:2:5 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $INCLUDE $CAMERA" 152 # convert 'stdout' to book format 153 ipptool2book stdout detPendingResidExp -key det_id:iteration:exp_tag -uniq 154 if ($VERBOSE > 2) 155 book listbook detPendingResidExp 225 156 end 226 157 227 158 # delete existing entries which are DONE 228 queuedrop DetrendResidExposures -key 0 DONE 229 end 230 231 # locked list 159 book delpage detPendingResidExp -key state DONE 160 end 161 232 162 # default exit status 233 163 task.exit default … … 248 178 249 179 task.exec 250 queuesize DetrendResidExposures-var N180 book npages detPendingResidExp -var N 251 181 if ($N == 0) break 252 182 if ($network == 0) break 253 183 254 # look for new images on the internal DetrendResidExposures queue 255 # caution with these 'if' statements: syntax errors 256 # will make the task fail without given a good status 257 queuepop DetrendResidExposures -var line -key 0 NEW 258 if ("$line" == "NULL") break 259 260 strpop line state 261 queuepush DetrendResidExposures -replace -key 1:2:5 "RUN $line" 262 263 ## the DetrendResidExposures queue contains: 264 ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN 265 ## NOTE: the queue has the STATE prepended 266 list word -split $line 267 $DET_ID = $word:0 268 $ITER = $word:1 269 $DET_TYPE = $word:2 270 $MODE = $word:3 271 $EXP_TAG = $word:4 272 $INCLUDE = $word:5 273 $CAMERA = $word:6 274 275 # XXX this is still a lame rule 276 $word = `basename $EXP_TAG | tr '.' ' '` 277 list word -split $word 278 $base = $word:0 279 280 stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log 281 stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log 282 exec mkdir -p $LOGSUBDIR/$base 184 # look for new images in detPendingResidExp 185 book getpage detPendingResidExp 0 -var pageName -key state NULL 186 if ($pageName == NULL) break 187 188 book setword detPendingResidExp $pageName state RUN 189 book getword detPendingResidExp $pageName det_id -var DET_ID 190 book getword detPendingResidExp $pageName iteration -var ITERATION 191 book getword detPendingResidExp $pageName det_type -var DET_TYPE 192 book getword detPendingResidExp $pageName mode -var MODE 193 book getword detPendingResidExp $pageName exp_tag -var EXP_TAG 194 book getword detPendingResidExp $pageName include -var INCLUDE 195 book getword detPendingResidExp $pageName camera -var CAMERA 283 196 284 197 # specify choice of remote host: … … 289 202 end 290 203 204 # XXX use ipp_filename.pl to lookup output file names 205 $word = `basename $EXP_TAG | tr '.' ' '` 206 list word -split $word 207 $base = $word:0 208 stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.log 209 stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.log 210 exec mkdir -p $LOGSUBDIR/$base 211 212 # save the pageName for future reference below 213 options $pageName 214 291 215 # create example job options as a demonstration 292 options "$line"293 216 if ($VERBOSE > 1) 294 echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA 295 end 296 command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA 297 end 298 299 # success 300 task.exit 0 301 queuepush DetrendResidExposures -replace -key 1:2:5 "DONE $options:0" 302 end 303 304 # default exit status 305 task.exit default 306 showcommand failure 307 queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0" 308 $DetrendResidExpFail ++ 309 end 310 311 # operation times out? 312 task.exit timeout 313 showcommand timeout 314 queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0" 315 $DetrendResidExpFail ++ 316 end 317 end 217 echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA 218 end 219 command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA 220 end 221 222 # default exit status 223 task.exit default 224 process_exit detPendingResidExp $options:0 $JOB_STATUS 225 end 226 227 # operation times out? 228 task.exit timeout 229 showcommand timeout 230 book setword detPendingResidExp $options:0 state TIMEOUT 231 end 232 end -
trunk/ippTasks/detrend.stack.pro
r10745 r11210 1 1 ## this file contains the tasks for running the detrend stacking stage 2 ## these tasks use the book detPendingStackedImfile 2 3 3 # get a list of imfiles to stack4 # dettool -tostack -simple5 # 1 0 bias mc6 # DET_ID ITER TYPE CLASS_ID7 8 # function to stack the imfiles:9 # detrend_stack.pl 1 0 mc bias10 11 ## these tasks use the queue DetrendStackClass12 ## the DetrendStackClass queue contains:13 ## STATE DET_ID ITER TYPE CLASS_ID CAMERA14 15 queueinit DetrendStackClass16 4 if ($?network == 0) 17 5 $network = 1 … … 20 8 $parallel = 0 21 9 end 22 23 $DetrendStackClassFail = 024 25 10 if ($?VERBOSE == 0) 26 11 echo "VERBOSE not defined: load pantasks.pro first" … … 31 16 break 32 17 end 18 33 19 $LOGSUBDIR = $LOGDIR/detrend 34 20 exec mkdir -p $LOGSUBDIR 35 21 22 book init detPendingStackedImfile 23 36 24 macro detstack.status 37 queueprint DetrendStackClass 38 echo "DetrendStackClassFail: $DetrendStackClassFail" 25 book listbook detPendingStackedImfile 39 26 end 40 27 41 # select images ready for copy42 # new entries are added to queue DetrendStackClass28 # select images ready for detrend_stack.pl 29 # new entries are added to detPendingStackedImfile 43 30 # compare the new list with the ones already selected 44 31 task dettool.stack.load … … 49 36 periods -timeout 30 50 37 51 stdout $LOGSUBDIR/detstack.log38 stdout NULL 52 39 stderr $LOGSUBDIR/detstack.log 53 40 54 41 task.exec 55 command dettool -tostacked - simple -limit {$DetrendStackClassFail + 20}42 command dettool -tostacked -limit 20 56 43 end 57 44 58 45 # success 59 46 task.exit 0 60 local i Nqueue 61 62 # compare output with newImage queue 63 # only add entries which do not exist in queue 64 queuesize stdout -var Nqueue 65 for i 0 $Nqueue 66 queuepop stdout -var line 67 if ($VERBOSE > 2) 68 echo $line 69 end 70 list word -split $line 71 $DET_ID = $word:0 72 $ITER = $word:1 73 $DET_TYPE = $word:2 74 $CLASS_ID = $word:3 75 $CAMERA = $word:4 76 queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $CLASS_ID $CAMERA" 47 # convert 'stdout' to book format 48 ipptool2book stdout detPendingStackedImfile -key det_id:iteration:class_id -uniq 49 if ($VERBOSE > 2) 50 book listbook detPendingStackedImfile 77 51 end 78 52 79 53 # delete existing entries which are DONE 80 queuedrop DetrendStackClass -key 0DONE54 book delpage detPendingStackedImfile -key state DONE 81 55 end 82 56 … … 99 73 100 74 task.exec 101 queuesize DetrendStackClass-var N75 book npages detPendingStackedImfile -var N 102 76 if ($N == 0) break 103 77 if ($network == 0) break 104 78 105 # look for new images on the internal DetrendStackClass queue 106 # caution with these 'if' statements: syntax errors 107 # will make the task fail without given a good status 108 queuepop DetrendStackClass -var line -key 0 NEW 109 if ("$line" == "NULL") break 79 # look for new images in detPendingStackedImfile 80 book getpage detPendingStackedImfile 0 -var pageName -key state NULL 81 if ($pageName == NULL) break 110 82 111 strpop line state 112 queuepush DetrendStackClass -replace -key 1:2:4 "RUN $line" 113 114 # the DetrendStackClass queue contains: 115 # STATE DET_ID ITER TYPE CLASS_ID 116 ## NOTE: the queue has the STATE prepended 117 # use local variables 118 list word -split $line 119 $DET_ID = $word:0 120 $ITER = $word:1 121 $DET_TYPE = $word:2 122 $CLASS_ID = $word:3 123 $CAMERA = $word:4 124 125 # XXX add $WORKDIR/$LOG_DIR 126 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log 127 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log 83 book setword detPendingStackedImfile $pageName state RUN 84 book getword detPendingStackedImfile $pageName det_id -var DET_ID 85 book getword detPendingStackedImfile $pageName iteration -var ITERATION 86 book getword detPendingStackedImfile $pageName det_type -var DET_TYPE 87 book getword detPendingStackedImfile $pageName class_id -var CLASS_ID 88 book getword detPendingStackedImfile $pageName camera -var CAMERA 128 89 129 90 # specify choice of remote host: … … 134 95 end 135 96 136 # create example job options as a demonstration 137 options "$line" 97 # XXX add $WORKDIR/$LOG_DIR 98 # XXX use ipp_filename.pl to lookup output file names 99 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.$CLASS_ID.log 100 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.$CLASS_ID.log 101 102 # save the pageName for future reference below 103 options $pageName 104 105 # create the command line 138 106 if ($VERBOSE > 1) 139 echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA107 echo command detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA 140 108 end 141 command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA 142 end 143 144 # success 145 task.exit 0 146 queuepush DetrendStackClass -replace -key 1:2:4 "DONE $options:0" 109 command detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA 147 110 end 148 111 149 112 # default exit status 150 113 task.exit default 151 showcommand failure 152 queuepush DetrendStackClass -replace -key 1:2:4 "FAIL $options:0" 153 $DetrendStackClassFail ++ 114 process_exit detPendingStackedImfile $options:0 $JOB_STATUS 154 115 end 155 116 … … 157 118 task.exit timeout 158 119 showcommand timeout 159 queuepush DetrendStackClass -replace -key 1:2:4 "TIMEOUT $options:0" 160 $DetrendStackClassFail ++ 120 book setword detPendingStackedImfile $options:0 state TIMEOUT 161 121 end 162 122 end -
trunk/ippTasks/notes.txt
r9032 r11210 1 2 2007.01.08 3 4 I have updated pantasks to used named tags in named groups to make 5 life much easier when changing the schema of the db tables. The 6 function ipptool2book convernts the output from an ippTool query in 7 the form of a set of metadata structures (saved in stdout) into a 8 book format. 9 10 dettool -tonormalizedstat DetrendNormStat 11 dettool -tonormalize DetrendNorm 12 dettool -tonormalizedexp DetrendNormExp 13 dettool -toprocessedimfile DetrendProcessImfiles 14 dettool -toprocessedexp DetrendProcessExposures 15 dettool -residdetrun DetrendRejectExp 16 dettool -toresidimfile DetrendResidImfiles 17 dettool -toresidexp DetrendResidExposures 18 dettool -tostacked DetrendStackClass 19 p0tool -pendingimfile Phase0Imfiles 20 p0tool -pendingexp Phase0Exposures 21 p2tool -pendingimfile Phase2Imfiles 22 p3tool -pendingexp Phase3Exposures 1 23 2 24 2006.08.28 -
trunk/ippTasks/pantasks.pro
r10760 r11210 18 18 $runexec = 1 19 19 20 $EXIT_SUCCESS = 0 21 $EXIT_UNKNOWN_ERR = 1 22 $EXIT_SYS_ERR = 2 23 $EXIT_CONFIG_ERR = 3 24 $EXIT_PROG_ERR = 4 25 $EXIT_DATA_ERR = 5 26 27 $DB:n = 0 28 29 macro init.simtest 30 $LOGDIR = `ipp_datapath.pl path://SIMTEST` 31 $LOGDIR = $LOGDIR/logs 32 exec mkdir -p $LOGDIR 33 34 list DB -add simtest 35 36 if ($parallel) 37 controller exit true 38 controller host add alala 39 end 40 end 41 20 42 macro init.essence 21 43 $LOGDIR = `ipp_datapath.pl path://ESSENCE` 22 44 $LOGDIR = $LOGDIR/logs 23 45 exec mkdir -p $LOGDIR 46 47 list DB -add essence_v2 24 48 25 49 if ($parallel) … … 36 60 $LOGDIR = $LOGDIR/logs 37 61 exec mkdir -p $LOGDIR 62 63 list DB -add isptest 38 64 39 65 if ($parallel) … … 55 81 macro load.tasks 56 82 input $scripts/phase0.pro 57 input $scripts/phase2.pro83 # input $scripts/phase2.pro 58 84 # input $scripts/phase3.pro 59 85 … … 75 101 if ($0 == 2) 76 102 echo "$1 for: $command" 103 echo "job exit status: $JOB_STATUS" 104 echo "job dtime: $JOB_DTIME" 77 105 else 78 106 echo "command: $command" … … 80 108 end 81 109 end 110 111 macro process_exit 112 if ($0 != 4) 113 echo "USAGE: process_exit (bookName) (pageName) (exitCode)" 114 break 115 end 116 117 $bookName = $1 118 $pageName = $2 119 $exitCode = $3 120 121 # success 122 if ($exitCode == $EXIT_SUCCESS) 123 # the handler scripts update DB the tables; here we just update the page 124 book setword $bookName $pageName state DONE 125 return 126 end 127 128 # failure related to the data files 129 # jobs which result in DATAERR must have db state updated 130 if ($exitCode == $EXIT_DATA_ERR) 131 showcommand failure 132 book setword $bookName $pageName state DATA_ERR 133 return 134 end 135 136 # failure related to the data files 137 if ($exitCode == $EXIT_SYS_ERR) 138 halt 139 showcommand "system failure" 140 book setword $bookName $pageName state SYS_ERR 141 return 142 end 143 144 # failure related to the data files 145 if ($exitCode == $EXIT_CONFIG_ERR) 146 halt 147 showcommand "config error" 148 book setword $bookName $pageName state CONFIG_ERR 149 return 150 end 151 152 # failure related to the data files 153 if ($exitCode == $EXIT_PROG_ERR) 154 halt 155 showcommand "programming error" 156 book setword $bookName $pageName state PROG_ERR 157 return 158 end 159 160 # any other exit status 161 showcommand failure 162 book setword $bookName $pageName state UNKNOWN_ERR 163 end -
trunk/ippTasks/phase0.pro
r10746 r11210 1 1 ## this file contains the tasks for running the phase 0 stage 2 3 ## these tasks use the queue Phase0Imfiles 4 ## the Phase0Imfiles queue contains: 5 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS 6 ## queue keys are counted from 0 7 8 queueinit Phase0Imfiles 9 queueinit Phase0Exposures 2 ## these tasks use the books p0PendingImfile and p0PendingExp 3 10 4 if ($?network == 0) 11 5 $network = 1 … … 14 8 $parallel = 0 15 9 end 16 17 $Phase0ImfileFail = 018 $Phase0ExpFail = 019 20 10 if ($?VERBOSE == 0) 21 11 echo "VERBOSE not defined: load pantasks.pro first" … … 26 16 break 27 17 end 18 28 19 $LOGSUBDIR = $LOGDIR/phase0 29 20 exec mkdir -p $LOGSUBDIR 30 21 22 book init p0PendingImfile 23 book init p0PendingExp 24 31 25 macro phase0.status 32 queueprint Phase0Imfiles33 queueprint Phase0Exposures34 echo "Phase0ImfileFail: $Phase0ImfileFail" 35 echo "Phase0ExpFail: $Phase0ExpFail" 36 end 26 book listbook p0PendingImfile 27 book listbook p0PendingExp 28 end 29 30 $phase0_DB = 0 37 31 38 32 # select images ready for phase0 analysis 39 # new entries are added to queue Phase0Imfiles33 # new entries are added to p0PendingImfile 40 34 # compare the new list with the ones already selected 41 35 task phase0.imfile.load … … 46 40 periods -timeout 30 47 41 48 stdout $LOGSUBDIR/phase0.imfile.log 42 # XXX test that this option works 43 npending 1 44 45 # silently drop stdout 46 stdout NULL 49 47 stderr $LOGSUBDIR/phase0.imfile.log 50 48 51 task.exec 52 command p0tool -pendingimfile -simple -limit {$Phase0ImfileFail + 20} 49 # select entries from the current DB; cycle to the next DB, if it exists 50 # if the DB list is not set, use the value defined in .ipprc 51 task.exec 52 if ($DB:n == 0) 53 command p0tool -pendingimfile -limit 20 54 else 55 command p0tool -pendingimfile -limit 20 -dbname $DB:$phase0_DB 56 $phase0_DB ++ 57 if ($phase0_DB >= $DB:n) set phase0_DB = 0 58 end 53 59 end 54 60 55 61 # success 56 task.exit 0 57 local i Nqueue 58 59 # compare output with Phase0Imfiles queue 60 # only add entries which don't exist in queue 61 queuesize stdout -var Nqueue 62 for i 0 $Nqueue 63 queuepop stdout -var line 64 if ($VERBOSE > 2) 65 echo $line 66 end 67 list word -split $line 68 $EXP_TAG = $word:0 69 $CLASS = $word:1 70 $CLASS_ID = $word:2 71 $URI = $word:3 72 queuepush Phase0Imfiles -uniq -key 1 "NEW $EXP_TAG $CLASS $CLASS_ID $URI" 73 end 74 75 # delete existing entries which are DONE 76 queuedrop Phase0Imfiles -key 0 DONE 62 task.exit $EXIT_SUCCESS 63 # convert 'stdout' to book format 64 ipptool2book stdout p0PendingImfile -key exp_tag -uniq 65 if ($VERBOSE > 2) 66 book listbook p0PendingImfile 67 end 68 69 # delete existing entries which are DONE or DATAERR 70 book delpage p0PendingImfile -key state DONE 71 book delpage p0PendingImfile -key state DATA_ERR 77 72 end 78 73 … … 88 83 end 89 84 90 91 92 # run the phase0imfile script on pending images 85 # run the phase0_imfile.pl script on pending images 93 86 task phase0.imfile.run 94 87 periods -poll $runpoll … … 97 90 98 91 task.exec 99 queuesize Phase0Imfiles-var N92 book npages p0PendingImfile -var N 100 93 if ($N == 0) break 101 94 if ($network == 0) break 102 95 103 # look for new images on the internal DetrendImfilesToProcess queue 104 # caution with these 'if' statements: syntax errors 105 # will make the task fail without given a good status 106 queuepop Phase0Imfiles -var line -key 0 NEW 107 if ("$line" == "NULL") break 108 109 strpop line state 110 queuepush Phase0Imfiles -replace -key 1 "RUN $line" 111 112 ## the Phase0Imfiles queue contains (line has popped STATE) 113 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS 114 list word -split $line 115 $EXP_TAG = $word:0 116 $CLASS = $word:1 117 $CLASS_ID = $word:2 118 $URI = $word:3 96 # look for new images in p0PendingImfile 97 book getpage p0PendingImfile 0 -var pageName -key state NULL 98 if ($pageName == NULL) break 99 100 book setword p0PendingImfile $pageName state RUN 101 102 book getword p0PendingImfile $pageName exp_tag -var EXP_TAG 103 book getword p0PendingImfile $pageName class -var CLASS 104 book getword p0PendingImfile $pageName class_id -var CLASS_ID 105 book getword p0PendingImfile $pageName uri -var URI 106 book getword p0PendingImfile $pageName outpath -var OUTPATH 107 book getword p0PendingImfile $pageName dbname -var DBNAME 108 109 # specify choice of remote host 110 if ($parallel) 111 host anyhost 112 else 113 host local 114 end 115 116 ## generate output log based on filerule 117 $outroot = $OUTPATH/$EXP_TAG 118 $logfile = `ipp_filename.pl PHASE0.LOG $outroot $CLASS_ID` 119 stdout $logfile 120 stderr $logfile 121 exec mkdir -p $OUTPATH 122 123 # save the pageName for future reference below 124 options $pageName 125 126 # create the command line 127 if ($VERBOSE > 1) 128 echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --dbname $DBNAME 129 end 130 command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --dbname $DBNAME 131 end 132 133 # default exit status 134 task.exit default 135 process_exit p0PendingImfile $options:0 $JOB_STATUS 136 end 137 138 # operation timed out? 139 task.exit timeout 140 showcommand timeout 141 book setword p0PendingImfile $options:0 state TIMEOUT 142 end 143 end 144 145 # select exposures ready for phase0_exp.pl 146 task phase0.exp.load 147 host local 148 149 periods -poll $loadpoll 150 periods -exec $loadexec 151 periods -timeout 30 152 153 stdout NULL 154 stderr $LOGSUBDIR/phase0.exp.log 155 156 task.exec 157 command p0tool -pendingexp -limit 20 158 end 159 160 # success 161 task.exit 0 162 # convert 'stdout' to book format 163 ipptool2book stdout p0PendingExp -key exp_tag -uniq 164 if ($VERBOSE > 2) 165 book listbook p0PendingExp 166 end 167 168 # delete existing entries which are DONE 169 book delpage p0PendingExp -key state DONE 170 book delpage p0PendingExp -key state DATA_ERR 171 end 172 173 # default exit status 174 task.exit default 175 showcommand failure 176 end 177 178 # operation times out? 179 task.exit timeout 180 showcommand timeout 181 end 182 end 183 184 # run the phase0_exp.pl script on pending exposures 185 task phase0.exp.run 186 periods -poll $runpoll 187 periods -exec $runexec 188 periods -timeout 30 189 190 task.exec 191 book npages p0PendingExp -var N 192 if ($N == 0) break 193 if ($network == 0) break 194 195 # look for new images in p0PendingExp (state == NULL) 196 book getpage p0PendingExp 0 -var pageName -key state NULL 197 if ($pageName == NULL) break 198 199 book setword p0PendingExp $pageName state RUN 200 book getword p0PendingExp $pageName exp_tag -var EXP_TAG 201 book getword p0PendingExp $pageName outpath -var OUTPATH 119 202 120 203 # specify choice of remote host:(need to choose based on chips) … … 125 208 end 126 209 127 # XXX this is still a lame rule 128 $word = `basename $EXP_TAG | tr '.' ' '` 129 list word -split $word 130 $base = $word:0 131 stdout $LOGSUBDIR/$base/$EXP_TAG.p0.log 132 stderr $LOGSUBDIR/$base/$EXP_TAG.p0.log 133 exec mkdir -p $LOGSUBDIR/$base 210 ## generate output log based on filerule 211 $outroot = $OUTPATH/$EXP_TAG 212 $logfile = `ipp_filename.pl PHASE0.LOG $outroot FPA` 213 stdout $logfile 214 stderr $logfile 215 exec mkdir -p $OUTPATH 216 217 # save the pageName for future reference below 218 options $pageName 134 219 135 220 # create the command line 136 options "$line"137 if ($VERBOSE > 1)138 echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI139 end140 command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI141 end142 143 # success144 task.exit 0145 # phase0imfile.pl updates DB tables, here we just update the queue146 queuepush Phase0Imfiles -replace -key 1 "DONE $options:0"147 end148 149 # default exit status150 task.exit default151 showcommand failure152 queuepush Phase0Imfiles -replace -key 1 "FAIL $options:0"153 $Phase0ImfileFail ++154 end155 156 # operation times out?157 task.exit timeout158 showcommand timeout159 queuepush Phase0Imfiles -replace -key 1 "TIMEOUT $options:0"160 $Phase0ImfileFail ++161 end162 end163 164 165 166 ## these tasks use the queue Phase0Exposures167 ## the Phase0Exposures queue contains:168 ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS169 170 # select images ready for phase0 analysis171 # new entries are added to queue Phase0Exposures172 # compare the new list with the ones already selected173 task phase0.exp.load174 host local175 176 periods -poll $loadpoll177 periods -exec $loadexec178 periods -timeout 30179 180 stdout $LOGSUBDIR/phase0.exp.log181 stderr $LOGSUBDIR/phase0.exp.log182 183 task.exec184 command p0tool -pendingexp -simple -limit {$Phase0ExpFail + 20}185 end186 187 # success188 task.exit 0189 local i Nqueue190 191 # compare output with Phase0Exposures queue192 # only add entries which don't exist in queue193 queuesize stdout -var Nqueue194 for i 0 $Nqueue195 queuepop stdout -var line196 if ($VERBOSE > 2)197 echo $line198 end199 list word -split $line200 $EXP_TAG = $word:0201 # $EXP_ID = $word:1202 # $CAMERA = $word:2203 # $TELESCOPE = $word:3204 # $DATEOBS = $word:4205 # $TYPE = $word:5206 # $IMFILES = $word:6207 queuepush Phase0Exposures -uniq -key 1 "NEW $EXP_TAG"208 end209 210 # delete existing entries which are DONE211 queuedrop Phase0Exposures -key 0 DONE212 end213 214 # default exit status215 task.exit default216 showcommand217 end218 219 # operation times out?220 task.exit timeout221 showcommand222 end223 end224 225 226 227 # run the phase0imfile script on pending images228 task phase0.exp.run229 periods -poll $runpoll230 periods -exec $runexec231 periods -timeout 30232 233 task.exec234 queuesize Phase0Exposures -var N235 if ($N == 0) break236 if ($network == 0) break237 238 # look for new images on the internal DetrendImfilesToProcess queue239 # caution with these 'if' statements: syntax errors240 # will make the task fail without given a good status241 queuepop Phase0Exposures -var line -key 0 NEW242 if ("$line" == "NULL") break243 244 strpop line state245 queuepush Phase0Exposures -replace -key 1 "RUN $line"246 247 ## the Phase0Exposures queue contains:248 ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS249 list word -split $line250 $EXP_TAG = $word:0251 252 # XXX this is still a lame rule253 $word = `basename $EXP_TAG | tr '.' ' '`254 list word -split $word255 $base = $word:0256 stdout $LOGSUBDIR/$base/$EXP_TAG.p0.log257 stderr $LOGSUBDIR/$base/$EXP_TAG.p0.log258 exec mkdir -p $LOGSUBDIR/$base259 260 # specify choice of remote host:(need to choose based on chips)261 if ($parallel)262 host anyhost263 else264 host local265 end266 267 # create the command line268 options "$line"269 221 if ($VERBOSE > 1) 270 222 echo command phase0_exp.pl --exp_tag $EXP_TAG … … 274 226 275 227 # success 276 task.exit 0 277 # phase0exp.pl updates DB tables, here we just update the queue 278 queuepush Phase0Exposures -replace -key 1 "DONE $options:0" 279 end 280 281 # default exit status 282 task.exit default 283 showcommand 284 queuepush Phase0Exposures -replace -key 1 "FAIL $options:0" 285 $Phase0ExpFail 228 task.exit default 229 process_exit p0PendingExp $options:0 $JOB_STATUS 286 230 end 287 231 288 232 # operation times out? 289 233 task.exit timeout 290 showcommand 291 queuepush Phase0Exposures -replace -key 1 "TIMEOUT $options:0" 292 $Phase0ExpFail 234 showcommand timeout 235 book setword p0PendingExp $options:0 state TIMEOUT 293 236 end 294 237 end -
trunk/ippTasks/phase2.pro
r10758 r11210 1 1 ## this file contains the tasks for running the phase 0 stage 2 ## these tasks use the book p2PendingImfile 2 3 3 ## these tasks use the queue Phase2Imfiles4 ## the Phase2Imfiles queue contains:5 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS6 ## queue keys are counted from 07 8 queueinit Phase2Imfiles9 4 if ($?network == 0) 10 5 $network = 1 … … 13 8 $parallel = 0 14 9 end 15 16 $Phase2ImfileFail = 017 18 10 if ($?VERBOSE == 0) 19 11 echo "VERBOSE not defined: load pantasks.pro first" … … 24 16 break 25 17 end 18 26 19 $LOGSUBDIR = $LOGDIR/phase2 27 20 exec mkdir -p $LOGSUBDIR 28 21 22 book init p2PendingImfile 23 29 24 macro phase2.status 30 queueprint Phase2Imfiles 31 echo "Phase2ImfileFail: $Phase2ImfileFail" 25 book listbook p2PendingImfile 32 26 end 33 27 34 28 # select images ready for phase2 analysis 35 # new entries are added to queue Phase2Imfiles36 # compare the new list with the ones already selected29 # new entries are added to p2PendingImfile 30 # skip already-present entries 37 31 task phase2.imfile.load 38 32 host local … … 42 36 periods -timeout 30 43 37 44 stdout $LOGSUBDIR/phase2.imfile.log38 stdout NULL 45 39 stderr $LOGSUBDIR/phase2.imfile.log 46 40 47 41 task.exec 48 # command p2tool -pendingimfile -simple -limit {$Phase2ImfileFail + 20} 49 command p2tool -pendingimfile -simple 42 command p2tool -pendingimfile -limit 20 50 43 end 51 44 52 45 # success 53 46 task.exit 0 54 local i Nqueue 55 56 # save stdout and stderr 57 58 # compare output with Phase2Imfiles queue 59 # only add entries which don't exist in queue 60 queuesize stdout -var Nqueue 61 for i 0 $Nqueue 62 queuepop stdout -var line 63 if ($VERBOSE > 2) 64 echo $line 65 end 66 list word -split $line 67 $EXP_TAG = $word:0 68 $CLASS_ID = $word:1 69 $URI = $word:2 70 queuepush Phase2Imfiles -uniq -key 1:2 "NEW $EXP_TAG $CLASS_ID $URI" 47 # convert 'stdout' to book format 48 ipptool2book stdout p2PendingImfile -key exp_tag:class_id -uniq 49 if ($VERBOSE > 2) 50 book listbook p2PendingImfile 71 51 end 72 52 73 53 # delete existing entries which are DONE 74 queuedrop Phase2Imfiles -key 0DONE54 book delpage p2PendingImfile -key state DONE 75 55 end 76 56 … … 86 66 end 87 67 88 # run the phase2 imfilescript on pending images68 # run the phase2.pl script on pending images 89 69 task phase2.imfile.run 90 70 periods -poll $runpoll … … 93 73 94 74 task.exec 95 queuesize Phase2Imfiles-var N75 book npages p2PendingImfile -var N 96 76 if ($N == 0) break 97 77 if ($network == 0) break 98 78 99 # look for new images on the internal DetrendImfilesToProcess queue 100 # caution with these 'if' statements: syntax errors 101 # will make the task fail without given a good status 102 queuepop Phase2Imfiles -var line -key 0 NEW 103 if ("$line" == "NULL") break 79 # look for new images in p2PendingImfile (state == NULL) 80 book getpage p2PendingImfile 0 -var pageName -key state NULL 81 if ($pageName == NULL) break 104 82 105 strpop line state 106 queuepush Phase2Imfiles -replace -key 1:2 "RUN $line" 83 book setword p2PendingImfile $pageName state RUN 84 book getword p2PendingImfile $pageName camera -var CAMERA 85 book getword p2PendingImfile $pageName exp_tag -var EXP_TAG 86 book getword p2PendingImfile $pageName outpath -var OUTPATH 87 book getword p2PendingImfile $pageName class_id -var CLASS_ID 88 book getword p2PendingImfile $pageName uri -var URI 107 89 108 ## the Phase2Imfiles queue contains (line has popped STATE) 109 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS 110 list word -split $line 111 $EXP_TAG = $word:0 112 $CLASS_ID = $word:1 113 $URI = $word:2 114 115 # specify choice of remote host:(need to choose based on chips) 90 # specify choice of remote host 91 # XXX need to choose based on chips 116 92 if ($parallel) 117 93 host anyhost … … 120 96 end 121 97 122 ## add 'p2' to the log files? 123 # XXX this is still a lame rule 124 $word = `basename $EXP_TAG | tr '.' ' '` 125 list word -split $word 126 $base = $word:0 127 stdout $LOGSUBDIR/$base/$EXP_TAG.$CLASS_ID.log 128 stderr $LOGSUBDIR/$base/$EXP_TAG.$CLASS_ID.log 129 exec mkdir -p $LOGSUBDIR/$base 98 ## generate output log based on filerule 99 $outroot = $OUTPATH/$EXP_TAG 100 $logfile = `ipp_filename.pl PHASE2.LOG $outroot $CLASS_ID` 101 stdout $logfile 102 stderr $logfile 103 exec mkdir -p $OUTPATH 104 105 # save the pageName for future reference below 106 options $pageName 130 107 131 108 # create the command line 132 options "$line"133 109 if ($VERBOSE > 1) 134 echo command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI 110 echo command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --camera $CAMERA 135 111 end 136 ## XXX this is annoying: phase2.pl now requires camera :: it should not need this, it has the file. 137 command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --camera CTIO_MOSAIC2 138 end 139 140 # success 141 task.exit 0 142 # phase2imfile.pl updates DB tables, here we just update the queue 143 queuepush Phase2Imfiles -replace -key 1:2 "DONE $options:0" 112 command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --camera $CAMERA --outpath $OUTPATH 144 113 end 145 114 146 115 # default exit status 147 116 task.exit default 148 showcommand failure 149 queuepush Phase2Imfiles -replace -key 1:2 "FAIL $options:0" 150 $Phase2ImfileFail ++ 117 process_exit p2PendingImfile $options:0 $JOB_STATUS 151 118 end 152 119 153 # operation time sout?120 # operation timed out? 154 121 task.exit timeout 155 122 showcommand timeout 156 queuepush Phase2Imfiles -replace -key 1:2 "TIMEOUT $options:0" 157 $Phase2ImfileFail ++ 123 book setword p2PendingImfile $options:0 state TIMEOUT 158 124 end 159 125 end -
trunk/ippTasks/phase3.pro
r10758 r11210 1 ## this file contains the tasks for running the phase 0 stage 1 ## this file contains the tasks for running the phase 3 stage 2 ## these tasks use the book p3PendingExp 2 3 3 ## these tasks use the queue Phase3Exposures4 ## the Phase3Exposures queue contains:5 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS6 ## queue keys are counted from 07 8 queueinit Phase3Exposures9 4 if ($?network == 0) 10 5 $network = 1 … … 13 8 $parallel = 0 14 9 end 15 16 $Phase3ExposureFail = 017 18 10 if ($?VERBOSE == 0) 19 11 echo "VERBOSE not defined: load pantasks.pro first" … … 24 16 break 25 17 end 18 26 19 $LOGSUBDIR = $LOGDIR/phase3 27 20 exec mkdir -p $LOGSUBDIR 28 21 22 book init p3PendingExp 23 29 24 macro phase3.status 30 queueprint Phase3Exposures 31 echo "Phase3ExposureFail: $Phase3ExposureFail" 25 book listbook p3PendingExp 32 26 end 33 27 34 28 # select images ready for phase3 analysis 35 # new entries are added to queue Phase3Exposures36 # compare the new list with the ones already selected29 # new entries are added to p3PendingExp 30 # skip already-present entries 37 31 task phase3.exp.load 38 32 host local … … 46 40 47 41 task.exec 48 command p3tool -pendingexp -simple42 command p3tool -pendingexp -limit 20 49 43 end 50 44 51 45 # success 52 46 task.exit 0 53 local i Nqueue 54 55 # compare output with Phase3Exposures queue 56 # only add entries which don't exist in queue 57 queuesize stdout -var Nqueue 58 for i 0 $Nqueue 59 queuepop stdout -var line 60 if ($VERBOSE > 2) 61 echo $line 62 end 63 list word -split $line 64 $EXP_TAG = $word:0 65 queuepush Phase3Exposures -uniq -key 1 "NEW $EXP_TAG" 47 # convert 'stdout' to book format 48 ipptool2book stdout p3PendingExp -key exp_tag -uniq 49 if ($VERBOSE > 2) 50 book listbook p3PendingExp 66 51 end 67 52 68 53 # delete existing entries which are DONE 69 queuedrop Phase3Exposures -key 0DONE54 book delpage p3PendingExp -key state DONE 70 55 end 71 56 … … 88 73 89 74 task.exec 90 queuesize Phase3Exposures-var N75 book npages p3PendingExp -var N 91 76 if ($N == 0) break 92 77 if ($network == 0) break 93 78 94 # look for new images on the internal DetrendExposuresToProcess queue 95 # caution with these 'if' statements: syntax errors 96 # will make the task fail without given a good status 97 queuepop Phase3Exposures -var line -key 0 NEW 98 if ("$line" == "NULL") break 79 # look for new images in p3PendingExp (state == NULL) 80 book getpage p3PendingExp 0 -var pageName -key state NULL 81 if ($pageName == NULL) break 99 82 100 strpop line state 101 queuepush Phase3Exposures -replace -key 1 "RUN $line" 83 book setword p3PendingExp $pageName state RUN 84 book getword p3PendingExp $pageName camera -var CAMERA 85 book getword p3PendingExp $pageName exp_tag -var EXP_TAG 102 86 103 ## the Phase3Exposures queue contains: 104 ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS 105 list word -split $line 106 $EXP_TAG = $word:0 87 # specify choice of remote host:(need to choose based on chips) 88 if ($parallel) 89 host anyhost 90 else 91 host local 92 end 107 93 108 94 # XXX this is still a lame rule … … 114 100 exec mkdir -p $LOGSUBDIR/$base 115 101 116 # specify choice of remote host:(need to choose based on chips) 117 if ($parallel) 118 host anyhost 119 else 120 host local 121 end 102 # save the pageName for future reference below 103 options $pageName 122 104 123 105 # create the command line 124 options "$line"125 106 if ($VERBOSE > 1) 126 107 echo command phase3.pl --exp_tag $EXP_TAG 127 108 end 128 command phase3.pl --exp_tag $EXP_TAG --camera CTIO_MOSAIC2109 command phase3.pl --exp_tag $EXP_TAG --camera $CAMERA 129 110 end 130 111 131 112 # success 132 113 task.exit 0 133 # phase3exp.pl updates DB tables, here we just update the queue134 queuepush Phase3Exposures -replace -key 1 "DONE $options:0"114 # phase3exp.pl updates DB tables, here we just update the page 115 book setword p3PendingExp $options:0 state DONE 135 116 end 136 117 … … 138 119 task.exit default 139 120 showcommand failure 140 queuepush Phase3Exposures -replace -key 1 "FAIL $options:0" 141 $Phase3ExposureFail ++ 121 book setword p3PendingExp $options:0 state FAIL 142 122 end 143 123 … … 145 125 task.exit timeout 146 126 showcommand timeout 147 queuepush Phase3Exposures -replace -key 1 "TIMEOUT $options:0" 148 $Phase3ExposureFail ++ 127 book setword p3PendingExp $options:0 state TIMEOUT 149 128 end 150 129 end
Note:
See TracChangeset
for help on using the changeset viewer.
