Changeset 11210 for trunk/ippTasks/phase0.pro
- Timestamp:
- Jan 19, 2007, 5:24:59 PM (19 years ago)
- File:
-
- 1 edited
-
trunk/ippTasks/phase0.pro (modified) (8 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippTasks/phase0.pro
r10746 r11210 1 1 ## this file contains the tasks for running the phase 0 stage 2 3 ## these tasks use the queue Phase0Imfiles 4 ## the Phase0Imfiles queue contains: 5 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS 6 ## queue keys are counted from 0 7 8 queueinit Phase0Imfiles 9 queueinit Phase0Exposures 2 ## these tasks use the books p0PendingImfile and p0PendingExp 3 10 4 if ($?network == 0) 11 5 $network = 1 … … 14 8 $parallel = 0 15 9 end 16 17 $Phase0ImfileFail = 018 $Phase0ExpFail = 019 20 10 if ($?VERBOSE == 0) 21 11 echo "VERBOSE not defined: load pantasks.pro first" … … 26 16 break 27 17 end 18 28 19 $LOGSUBDIR = $LOGDIR/phase0 29 20 exec mkdir -p $LOGSUBDIR 30 21 22 book init p0PendingImfile 23 book init p0PendingExp 24 31 25 macro phase0.status 32 queueprint Phase0Imfiles33 queueprint Phase0Exposures34 echo "Phase0ImfileFail: $Phase0ImfileFail" 35 echo "Phase0ExpFail: $Phase0ExpFail" 36 end 26 book listbook p0PendingImfile 27 book listbook p0PendingExp 28 end 29 30 $phase0_DB = 0 37 31 38 32 # select images ready for phase0 analysis 39 # new entries are added to queue Phase0Imfiles33 # new entries are added to p0PendingImfile 40 34 # compare the new list with the ones already selected 41 35 task phase0.imfile.load … … 46 40 periods -timeout 30 47 41 48 stdout $LOGSUBDIR/phase0.imfile.log 42 # XXX test that this option works 43 npending 1 44 45 # silently drop stdout 46 stdout NULL 49 47 stderr $LOGSUBDIR/phase0.imfile.log 50 48 51 task.exec 52 command p0tool -pendingimfile -simple -limit {$Phase0ImfileFail + 20} 49 # select entries from the current DB; cycle to the next DB, if it exists 50 # if the DB list is not set, use the value defined in .ipprc 51 task.exec 52 if ($DB:n == 0) 53 command p0tool -pendingimfile -limit 20 54 else 55 command p0tool -pendingimfile -limit 20 -dbname $DB:$phase0_DB 56 $phase0_DB ++ 57 if ($phase0_DB >= $DB:n) set phase0_DB = 0 58 end 53 59 end 54 60 55 61 # success 56 task.exit 0 57 local i Nqueue 58 59 # compare output with Phase0Imfiles queue 60 # only add entries which don't exist in queue 61 queuesize stdout -var Nqueue 62 for i 0 $Nqueue 63 queuepop stdout -var line 64 if ($VERBOSE > 2) 65 echo $line 66 end 67 list word -split $line 68 $EXP_TAG = $word:0 69 $CLASS = $word:1 70 $CLASS_ID = $word:2 71 $URI = $word:3 72 queuepush Phase0Imfiles -uniq -key 1 "NEW $EXP_TAG $CLASS $CLASS_ID $URI" 73 end 74 75 # delete existing entries which are DONE 76 queuedrop Phase0Imfiles -key 0 DONE 62 task.exit $EXIT_SUCCESS 63 # convert 'stdout' to book format 64 ipptool2book stdout p0PendingImfile -key exp_tag -uniq 65 if ($VERBOSE > 2) 66 book listbook p0PendingImfile 67 end 68 69 # delete existing entries which are DONE or DATAERR 70 book delpage p0PendingImfile -key state DONE 71 book delpage p0PendingImfile -key state DATA_ERR 77 72 end 78 73 … … 88 83 end 89 84 90 91 92 # run the phase0imfile script on pending images 85 # run the phase0_imfile.pl script on pending images 93 86 task phase0.imfile.run 94 87 periods -poll $runpoll … … 97 90 98 91 task.exec 99 queuesize Phase0Imfiles-var N92 book npages p0PendingImfile -var N 100 93 if ($N == 0) break 101 94 if ($network == 0) break 102 95 103 # look for new images on the internal DetrendImfilesToProcess queue 104 # caution with these 'if' statements: syntax errors 105 # will make the task fail without given a good status 106 queuepop Phase0Imfiles -var line -key 0 NEW 107 if ("$line" == "NULL") break 108 109 strpop line state 110 queuepush Phase0Imfiles -replace -key 1 "RUN $line" 111 112 ## the Phase0Imfiles queue contains (line has popped STATE) 113 ## STATE EXP_TAG CLASS CLASS_ID URI-FITS 114 list word -split $line 115 $EXP_TAG = $word:0 116 $CLASS = $word:1 117 $CLASS_ID = $word:2 118 $URI = $word:3 96 # look for new images in p0PendingImfile 97 book getpage p0PendingImfile 0 -var pageName -key state NULL 98 if ($pageName == NULL) break 99 100 book setword p0PendingImfile $pageName state RUN 101 102 book getword p0PendingImfile $pageName exp_tag -var EXP_TAG 103 book getword p0PendingImfile $pageName class -var CLASS 104 book getword p0PendingImfile $pageName class_id -var CLASS_ID 105 book getword p0PendingImfile $pageName uri -var URI 106 book getword p0PendingImfile $pageName outpath -var OUTPATH 107 book getword p0PendingImfile $pageName dbname -var DBNAME 108 109 # specify choice of remote host 110 if ($parallel) 111 host anyhost 112 else 113 host local 114 end 115 116 ## generate output log based on filerule 117 $outroot = $OUTPATH/$EXP_TAG 118 $logfile = `ipp_filename.pl PHASE0.LOG $outroot $CLASS_ID` 119 stdout $logfile 120 stderr $logfile 121 exec mkdir -p $OUTPATH 122 123 # save the pageName for future reference below 124 options $pageName 125 126 # create the command line 127 if ($VERBOSE > 1) 128 echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --dbname $DBNAME 129 end 130 command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --dbname $DBNAME 131 end 132 133 # default exit status 134 task.exit default 135 process_exit p0PendingImfile $options:0 $JOB_STATUS 136 end 137 138 # operation timed out? 139 task.exit timeout 140 showcommand timeout 141 book setword p0PendingImfile $options:0 state TIMEOUT 142 end 143 end 144 145 # select exposures ready for phase0_exp.pl 146 task phase0.exp.load 147 host local 148 149 periods -poll $loadpoll 150 periods -exec $loadexec 151 periods -timeout 30 152 153 stdout NULL 154 stderr $LOGSUBDIR/phase0.exp.log 155 156 task.exec 157 command p0tool -pendingexp -limit 20 158 end 159 160 # success 161 task.exit 0 162 # convert 'stdout' to book format 163 ipptool2book stdout p0PendingExp -key exp_tag -uniq 164 if ($VERBOSE > 2) 165 book listbook p0PendingExp 166 end 167 168 # delete existing entries which are DONE 169 book delpage p0PendingExp -key state DONE 170 book delpage p0PendingExp -key state DATA_ERR 171 end 172 173 # default exit status 174 task.exit default 175 showcommand failure 176 end 177 178 # operation times out? 179 task.exit timeout 180 showcommand timeout 181 end 182 end 183 184 # run the phase0_exp.pl script on pending exposures 185 task phase0.exp.run 186 periods -poll $runpoll 187 periods -exec $runexec 188 periods -timeout 30 189 190 task.exec 191 book npages p0PendingExp -var N 192 if ($N == 0) break 193 if ($network == 0) break 194 195 # look for new images in p0PendingExp (state == NULL) 196 book getpage p0PendingExp 0 -var pageName -key state NULL 197 if ($pageName == NULL) break 198 199 book setword p0PendingExp $pageName state RUN 200 book getword p0PendingExp $pageName exp_tag -var EXP_TAG 201 book getword p0PendingExp $pageName outpath -var OUTPATH 119 202 120 203 # specify choice of remote host:(need to choose based on chips) … … 125 208 end 126 209 127 # XXX this is still a lame rule 128 $word = `basename $EXP_TAG | tr '.' ' '` 129 list word -split $word 130 $base = $word:0 131 stdout $LOGSUBDIR/$base/$EXP_TAG.p0.log 132 stderr $LOGSUBDIR/$base/$EXP_TAG.p0.log 133 exec mkdir -p $LOGSUBDIR/$base 210 ## generate output log based on filerule 211 $outroot = $OUTPATH/$EXP_TAG 212 $logfile = `ipp_filename.pl PHASE0.LOG $outroot FPA` 213 stdout $logfile 214 stderr $logfile 215 exec mkdir -p $OUTPATH 216 217 # save the pageName for future reference below 218 options $pageName 134 219 135 220 # create the command line 136 options "$line"137 if ($VERBOSE > 1)138 echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI139 end140 command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI141 end142 143 # success144 task.exit 0145 # phase0imfile.pl updates DB tables, here we just update the queue146 queuepush Phase0Imfiles -replace -key 1 "DONE $options:0"147 end148 149 # default exit status150 task.exit default151 showcommand failure152 queuepush Phase0Imfiles -replace -key 1 "FAIL $options:0"153 $Phase0ImfileFail ++154 end155 156 # operation times out?157 task.exit timeout158 showcommand timeout159 queuepush Phase0Imfiles -replace -key 1 "TIMEOUT $options:0"160 $Phase0ImfileFail ++161 end162 end163 164 165 166 ## these tasks use the queue Phase0Exposures167 ## the Phase0Exposures queue contains:168 ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS169 170 # select images ready for phase0 analysis171 # new entries are added to queue Phase0Exposures172 # compare the new list with the ones already selected173 task phase0.exp.load174 host local175 176 periods -poll $loadpoll177 periods -exec $loadexec178 periods -timeout 30179 180 stdout $LOGSUBDIR/phase0.exp.log181 stderr $LOGSUBDIR/phase0.exp.log182 183 task.exec184 command p0tool -pendingexp -simple -limit {$Phase0ExpFail + 20}185 end186 187 # success188 task.exit 0189 local i Nqueue190 191 # compare output with Phase0Exposures queue192 # only add entries which don't exist in queue193 queuesize stdout -var Nqueue194 for i 0 $Nqueue195 queuepop stdout -var line196 if ($VERBOSE > 2)197 echo $line198 end199 list word -split $line200 $EXP_TAG = $word:0201 # $EXP_ID = $word:1202 # $CAMERA = $word:2203 # $TELESCOPE = $word:3204 # $DATEOBS = $word:4205 # $TYPE = $word:5206 # $IMFILES = $word:6207 queuepush Phase0Exposures -uniq -key 1 "NEW $EXP_TAG"208 end209 210 # delete existing entries which are DONE211 queuedrop Phase0Exposures -key 0 DONE212 end213 214 # default exit status215 task.exit default216 showcommand217 end218 219 # operation times out?220 task.exit timeout221 showcommand222 end223 end224 225 226 227 # run the phase0imfile script on pending images228 task phase0.exp.run229 periods -poll $runpoll230 periods -exec $runexec231 periods -timeout 30232 233 task.exec234 queuesize Phase0Exposures -var N235 if ($N == 0) break236 if ($network == 0) break237 238 # look for new images on the internal DetrendImfilesToProcess queue239 # caution with these 'if' statements: syntax errors240 # will make the task fail without given a good status241 queuepop Phase0Exposures -var line -key 0 NEW242 if ("$line" == "NULL") break243 244 strpop line state245 queuepush Phase0Exposures -replace -key 1 "RUN $line"246 247 ## the Phase0Exposures queue contains:248 ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS249 list word -split $line250 $EXP_TAG = $word:0251 252 # XXX this is still a lame rule253 $word = `basename $EXP_TAG | tr '.' ' '`254 list word -split $word255 $base = $word:0256 stdout $LOGSUBDIR/$base/$EXP_TAG.p0.log257 stderr $LOGSUBDIR/$base/$EXP_TAG.p0.log258 exec mkdir -p $LOGSUBDIR/$base259 260 # specify choice of remote host:(need to choose based on chips)261 if ($parallel)262 host anyhost263 else264 host local265 end266 267 # create the command line268 options "$line"269 221 if ($VERBOSE > 1) 270 222 echo command phase0_exp.pl --exp_tag $EXP_TAG … … 274 226 275 227 # success 276 task.exit 0 277 # phase0exp.pl updates DB tables, here we just update the queue 278 queuepush Phase0Exposures -replace -key 1 "DONE $options:0" 279 end 280 281 # default exit status 282 task.exit default 283 showcommand 284 queuepush Phase0Exposures -replace -key 1 "FAIL $options:0" 285 $Phase0ExpFail 228 task.exit default 229 process_exit p0PendingExp $options:0 $JOB_STATUS 286 230 end 287 231 288 232 # operation times out? 289 233 task.exit timeout 290 showcommand 291 queuepush Phase0Exposures -replace -key 1 "TIMEOUT $options:0" 292 $Phase0ExpFail 234 showcommand timeout 235 book setword p0PendingExp $options:0 state TIMEOUT 293 236 end 294 237 end
Note:
See TracChangeset
for help on using the changeset viewer.
