IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 9104


Ignore:
Timestamp:
Oct 2, 2006, 12:36:59 PM (20 years ago)
Author:
eugene
Message:

updates to match new refactoring of tables

Location:
trunk/ippTasks
Files:
1 deleted
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/detrend.norm.pro

    r8193 r9104  
    22## this file contains the tasks for running the detrend normalization stage
    33
    4 ## these tasks use the queue DetrendRunToNormalize
     4## these tasks use the queue DetrendNormalize
     5## the DetrendNormalize queue contains:
     6## STATE DET_ID ITER
     7
     8queueinit DetrendNormalize
     9queueinit DetrendNormStat
     10queueinit DetrendNormalizeExposures
     11if ($?network == 0)
     12  $network = 1
     13end
     14if ($?parallel == 0)
     15  $parallel = 0
     16end
    517
    618# select images ready for copy
    7 # new entries are added to queue DetrendRunToNormalize
     19# new entries are added to queue DetrendNormalize
    820# compare the new list with the ones already selected
    921task           dettool.norm.load
    10   command      dettool -stacked -runs
     22  command      dettool -tonormalize -simple
    1123  host         local
    1224
    1325  periods      -poll 1
    1426  periods      -exec 5
    15   periods      -timeout 30
     27  periods      -timeout 5
     28
     29  stdout dettool.norm.log
     30  stderr dettool.norm.log
    1631
    1732  # success
    … …  
    2641    for i 0 $Nqueue
    2742      queuepop stdout -var line
    28       queuepush DetrendRunToNormalize -uniq -key 1:2:4 "NEW $line"
     43      queuepush DetrendNormalize -uniq -key 1:2 "NEW $line"
    2944    end
    3045
    3146    # delete existing entries which are DONE
    32     queuedrop DetrendRunToNormalize -key 0 DONE
     47    queuedrop DetrendNormalize -key 0 DONE
    3348  end
    3449
    3550  # locked list
    36   task.exit    1
    37     echo       "new.images: exec failure"
    38   end
    39 
    40   # default exit status
    41   task.exit    default
    42     echo       "new.images: unknown exit status: $EXIT"
    43   end
    44 
    45   # operation times out?
    46   task.exit    timeout
    47     echo       "new.images: timeout"
     51  # default exit status
     52  task.exit    default
     53    echo       "detrend.norm: failure"
     54  end
     55
     56  # operation times out?
     57  task.exit    timeout
     58    echo       "detrend.norm: timeout"
    4859  end
    4960end
    … …  
    5263task           dettool.norm.process
    5364  periods      -poll 0.5
    54   periods      -exec 1.0
    55   periods      -timeout 5
     65  periods      -exec 5.0
     66  periods      -timeout 30
    5667
    5768  task.exec
    58     queuesize DetrendRunToNormalize -var N
     69    queuesize DetrendNormalize -var N
    5970   
    6071    if ($N == 0) break
    6172    if ($network == 0) break
    6273   
    63     # look for new images on the internal DetrendRunToNormalize queue
     74    # look for new images on the internal DetrendNormalize queue
    6475    # caution with these 'if' statements: syntax errors
    6576    # will make the task fail without given a good status
    66     queuepop DetrendRunToNormalize -var line -key 0 NEW
     77    queuepop DetrendNormalize -var line -key 0 NEW
    6778    if ("$line" == "NULL") break
    6879
    6980    strpop line state
    70     queuepush DetrendRunToNormalize -replace -key 0 "RUN $line"
     81    queuepush DetrendNormalize -replace -key 1:2 "RUN $line"
     82
     83    # the DetrendNormalize queue contains:
     84    # STATE DET_ID ITER
     85    list word -split $line
     86    $DET_ID   = $word:0
     87    $ITER     = $word:1
     88
     89    # XXX we need to get CAMERA and TYPE from this query
     90    # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log
     91    # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log
     92    stdout isp.flat.$DET_ID.$ITER.log
     93    stderr isp.flat.$DET_ID.$ITER.log
    7194
    7295    # specify choice of remote host:
    73     # 'anyhost' specified random remote host
    74     # if we needed to specify a host explicitly, use a command like
    75     # one of the following.  the -required forces execution on the
    76     # named host, and blocks the task until the host is available
    77     # host kiawe
    78     # host alala -required
    79     host anyhost
    80 
    81     # choose the default recipe appropriate to this type of file
    82     $recipe = `get.recipe $type`
     96    if ($parallel)
     97      host anyhost
     98    else
     99      host local
     100    end
    83101
    84102    # create example job options as a demonstration
    85103    options "$line"
    86 
    87     # we can put the dettool and ppMerge commands here, or wrap them in a single script
    88     # $urilist = `dettool -stacked -det_id $det_id -iteration $iteration -chip"
    89     # $uri_out = ???
    90     # command ppNorm $uri_out $uri $uri ... -recipe PPMERGE $recipe ???
    91     command ppNorm $uri_out -det_id $det_id -iteration $iteration -recipe PPMERGE $recipe
     104    echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER
     105    command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER
    92106  end
    93107
    94108  # success
    95109  task.exit 0
    96     # XXX need to parse the options to identify the values below (if not part of cmdline?)
    97     exec dettool -addstacked -det_id $det_id -iteration $iteration -class_id $class_id -procrecipe $recipe
    98     queuepush DetrendRunToNormalize -replace -key 1:2:4 "DONE $options:0"
    99   end
    100 
    101   # default exit status
    102   task.exit    default
    103     queuepush DetrendRunToNormalize -replace -key 0 "FAIL $options:0"
    104   end
    105 
    106   # operation times out?
    107   task.exit    timeout
    108     queuepush DetrendRunToNormalize -replace -key 0 "FAIL $options:0"
    109   end
    110 end
     110    queuepush DetrendNormalize -replace -key 1:2 "DONE $options:0"
     111  end
     112
     113  # default exit status
     114  task.exit    default
     115    queuepush DetrendNormalize -replace -key 1:2 "FAIL $options:0"
     116  end
     117
     118  # operation times out?
     119  task.exit    timeout
     120    queuepush DetrendNormalize -replace -key 1:2 "TIMEOUT $options:0"
     121  end
     122end
     123
     124# select images ready for copy
     125# new entries are added to queue DetrendNormStat
     126# compare the new list with the ones already selected
     127task           dettool.normstat.load
     128  command      dettool -normstat -simple
     129  host         local
     130
     131  periods      -poll 1
     132  periods      -exec 5
     133  periods      -timeout 5
     134
     135  stdout dettool.normstat.log
     136  stderr dettool.normstat.log
     137
     138  # success
     139  task.exit    0
     140    # XXX is it necessary for these to be local?
     141    # drop after this task macro is done?
     142    local i Nqueue
     143
     144    # compare output with newImage queue
     145    # only add entries which don't exist in queue
     146    queuesize stdout -var Nqueue
     147    for i 0 $Nqueue
     148      queuepop stdout -var line
     149      queuepush DetrendNormStat -uniq -key 1:2 "NEW $line"
     150    end
     151
     152    # delete existing entries which are DONE
     153    queuedrop DetrendNormStat -key 0 DONE
     154  end
     155
     156  # locked list
     157  # default exit status
     158  task.exit    default
     159    echo       "detrend.norm: failure"
     160  end
     161
     162  # operation times out?
     163  task.exit    timeout
     164    echo       "detrend.norm: timeout"
     165  end
     166end
     167
     168# copy new images, sending job to desired host
     169task           dettool.normstat.process
     170  periods      -poll 0.5
     171  periods      -exec 5.0
     172  periods      -timeout 30
     173
     174  task.exec
     175    queuesize DetrendNormStat -var N
     176   
     177    if ($N == 0) break
     178    if ($network == 0) break
     179   
     180    # look for new images on the internal DetrendNormStat queue
     181    # caution with these 'if' statements: syntax errors
     182    # will make the task fail without given a good status
     183    queuepop DetrendNormStat -var line -key 0 NEW
     184    if ("$line" == "NULL") break
     185
     186    strpop line state
     187    queuepush DetrendNormStat -replace -key 1:2 "RUN $line"
     188
     189    # the DetrendNormStat queue contains:
     190    # STATE DET_ID ITER
     191    list word -split $line
     192    $DET_ID   = $word:0
     193    $ITER     = $word:1
     194
     195    # XXX we need to get CAMERA and TYPE from this query
     196    # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log
     197    # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log
     198    stdout isp.flat.$DET_ID.$ITER.log
     199    stderr isp.flat.$DET_ID.$ITER.log
     200
     201    # specify choice of remote host:
     202    if ($parallel)
     203      host anyhost
     204    else
     205      host local
     206    end
     207
     208    # create example job options as a demonstration
     209    options "$line"
     210    echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
     211    command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
     212  end
     213
     214  # success
     215  task.exit 0
     216    queuepush DetrendNormStat -replace -key 1:2 "DONE $options:0"
     217  end
     218
     219  # default exit status
     220  task.exit    default
     221    queuepush DetrendNormStat -replace -key 1:2 "FAIL $options:0"
     222  end
     223
     224  # operation times out?
     225  task.exit    timeout
     226    queuepush DetrendNormStat -replace -key 1:2 "TIMEOUT $options:0"
     227  end
     228end
     229
     230# select images ready for copy
     231# new entries are added to queue DetrendNormalizeExposures
     232# compare the new list with the ones already selected
     233task           dettool.normexp.load
     234  command      dettool -tonormalizedexp -simple
     235  host         local
     236
     237  periods      -poll 1
     238  periods      -exec 5
     239  periods      -timeout 5
     240
     241  stdout dettool.normexp.log
     242  stderr dettool.normexp.log
     243
     244  # success
     245  task.exit    0
     246    # XXX is it necessary for these to be local?
     247    # drop after this task macro is done?
     248    local i Nqueue
     249
     250    # compare output with newImage queue
     251    # only add entries which don't exist in queue
     252    queuesize stdout -var Nqueue
     253    for i 0 $Nqueue
     254      queuepop stdout -var line
     255      queuepush DetrendNormalizeExposures -uniq -key 1:2 "NEW $line"
     256    end
     257
     258    # delete existing entries which are DONE
     259    queuedrop DetrendNormalizeExposures -key 0 DONE
     260  end
     261
     262  # locked list
     263  # default exit status
     264  task.exit    default
     265    echo       "detrend.norm: failure"
     266  end
     267
     268  # operation times out?
     269  task.exit    timeout
     270    echo       "detrend.norm: timeout"
     271  end
     272end
     273
     274# copy new images, sending job to desired host
     275task           dettool.normexp.process
     276  periods      -poll 0.5
     277  periods      -exec 5.0
     278  periods      -timeout 30
     279
     280  task.exec
     281    queuesize DetrendNormalizeExposures -var N
     282   
     283    if ($N == 0) break
     284    if ($network == 0) break
     285   
     286    # look for new images on the internal DetrendNormalizeExposures queue
     287    # caution with these 'if' statements: syntax errors
     288    # will make the task fail without given a good status
     289    queuepop DetrendNormalizeExposures -var line -key 0 NEW
     290    if ("$line" == "NULL") break
     291
     292    strpop line state
     293    queuepush DetrendNormalizeExposures -replace -key 1:2 "RUN $line"
     294
     295    # the DetrendNormalizeExposures queue contains:
     296    # STATE DET_ID ITER
     297    list word -split $line
     298    $DET_ID   = $word:0
     299    $ITER     = $word:1
     300
     301    # XXX we need to get CAMERA and TYPE from this query
     302    # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log
     303    # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log
     304    stdout isp.flat.$DET_ID.$ITER.log
     305    stderr isp.flat.$DET_ID.$ITER.log
     306
     307    # specify choice of remote host:
     308    if ($parallel)
     309      host anyhost
     310    else
     311      host local
     312    end
     313
     314    # create example job options as a demonstration
     315    options "$line"
     316    echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
     317    command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
     318  end
     319
     320  # success
     321  task.exit 0
     322    queuepush DetrendNormalizeExposures -replace -key 1:2 "DONE $options:0"
     323  end
     324
     325  # default exit status
     326  task.exit    default
     327    queuepush DetrendNormalizeExposures -replace -key 1:2 "FAIL $options:0"
     328  end
     329
     330  # operation times out?
     331  task.exit    timeout
     332    queuepush DetrendNormalizeExposures -replace -key 1:2 "TIMEOUT $options:0"
     333  end
     334end

  • trunk/ippTasks/detrend.process.pro

    r9081 r9104  
    99# detrend_process.pl 1 850131b mc bias 850131b.fits 850131b
    1010
    11 ## these tasks use the queue DetrendImfilesToProcess
    12 ## the DetrendImfilesToProcess queue contains:
     11## these tasks use the queue DetrendProcessImfiles
     12## the DetrendProcessImfiles queue contains:
    1313## STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI
    1414
    15 queueinit DetrendImfilesToProcess
     15queueinit DetrendProcessImfiles
     16queueinit DetrendProcessExposures
    1617if ($?network == 0)
    1718  $network = 1
    … …  
    2223
    2324# select images ready for copy
    24 # new entries are added to queue DetrendImfilesToProcess
     25# new entries are added to queue DetrendProcessImfiles
    2526# compare the new list with the ones already selected
    26 task           dettool.raw.load
     27task           dettool.process.load
    2728  command      dettool -raw -simple
    2829  host         local
    … …  
    5354      $CLASS_ID = $word:4
    5455      $URI      = $word:5
    55       queuepush DetrendImfilesToProcess -uniq -key 1:3:5 "NEW $DET_ID $TYPE $EXP_KEY $CLASS $CLASS_ID $URI"
     56      queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $TYPE $EXP_KEY $CLASS $CLASS_ID $URI"
    5657    end
    5758
    5859    # delete existing entries which are DONE
    59     queuedrop DetrendImfilesToProcess -key 0 DONE
    60   end
    61 
    62   # locked list
    63   task.exit    1
    64     echo       "new.images: exec failure"
    65   end
    66 
    67   # default exit status
    68   task.exit    default
    69     echo       "new.images: unknown exit status: $EXIT"
    70   end
    71 
    72   # operation times out?
    73   task.exit    timeout
    74     echo       "new.images: timeout"
     60    queuedrop DetrendProcessImfiles -key 0 DONE
     61  end
     62
     63  # error
     64  task.exit    default
     65    echo       "detrend.process: failure"
     66  end
     67
     68  # operation times out?
     69  task.exit    timeout
     70    echo       "detrend.process: timeout"
    7571  end
    7672end
    7773
    7874# copy new images, sending job to desired host
    79 task           dettool.raw.process
     75task           dettool.process.run
    8076  periods      -poll 0.5
    8177  periods      -exec 2.0
    … …  
    8379
    8480  task.exec
    85     queuesize DetrendImfilesToProcess -var N
     81    queuesize DetrendProcessImfiles -var N
    8682    if ($N == 0) break
    8783    if ($network == 0) break
    8884   
    89     # look for new images on the internal DetrendImfilesToProcess queue
     85    # look for new images on the internal DetrendProcessImfiles queue
    9086    # caution with these 'if' statements: syntax errors
    9187    # will make the task fail without given a good status
    92     queuepop DetrendImfilesToProcess -var line -key 0 NEW
     88    queuepop DetrendProcessImfiles -var line -key 0 NEW
    9389    if ("$line" == "NULL") break
    9490
    9591    strpop line state
    96     queuepush DetrendImfilesToProcess -replace -key 1:3:5 "RUN $line"
    97 
    98     ## the DetrendImfilesToProcess queue contains:
     92    queuepush DetrendProcessImfiles -replace -key 1:3:5 "RUN $line"
     93
     94    ## the DetrendProcessImfiles queue contains:
    9995    # STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI
    10096    ## NOTE: the queue has the STATE prepended
    … …  
    119115    # create example job options as a demonstration
    120116    options "$line"
    121     echo command detrend_process.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI
    122     command detrend_process.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI
     117    echo command detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI
     118    command detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI
    123119  end
    124120
    125121  # success
    126122  task.exit 0
    127     queuepush DetrendImfilesToProcess -replace -key 1:3:5 "DONE $options:0"
     123    queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0"
    128124  end
    129125
    130126  # default exit status
    131127  task.exit    default
    132     queuepush DetrendImfilesToProcess -replace -key 1:3:5 "FAIL $options:0"
    133   end
    134 
    135   # operation times out?
    136   task.exit    timeout
    137     queuepush DetrendImfilesToProcess -replace -key 1:3:5 "TIMEOUT $options:0"
    138   end
    139 end
     128    queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0"
     129  end
     130
     131  # operation times out?
     132  task.exit    timeout
     133    queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0"
     134  end
     135end
     136
     137# select images ready for copy
     138# new entries are added to queue DetrendProcessExposures
     139# compare the new list with the ones already selected
     140task           dettool.processexp.load
     141  command      dettool -toprocessdexp -simple
     142  host         local
     143
     144  periods      -poll 1
     145  periods      -exec 5
     146  periods      -timeout 30
     147
     148  stdout dettool.processexp.log
     149  stderr dettool.processexp.log
     150
     151  # success
     152  task.exit    0
     153    # XXX is it necessary for these to be local?
     154    # drop after this task macro is done?
     155    local i Nqueue
     156
     157    # compare output with newImage queue
     158    # only add entries which don't exist in queue
     159    queuesize stdout -var Nqueue
     160    for i 0 $Nqueue
     161      queuepop stdout -var line
     162      list word -split $line
     163      $DET_ID   = $word:0
     164      $TYPE     = $word:1
     165      $EXP_KEY  = $word:2
     166      queuepush DetrendProcessExposures -uniq -key 1:3 "NEW $DET_ID $TYPE $EXP_KEY"
     167    end
     168
     169    # delete existing entries which are DONE
     170    queuedrop DetrendProcessExposures -key 0 DONE
     171  end
     172
     173  # error
     174  task.exit    default
     175    echo       "detrend.process: failure"
     176  end
     177
     178  # operation times out?
     179  task.exit    timeout
     180    echo       "detrend.process: timeout"
     181  end
     182end
     183
     184# copy new images, sending job to desired host
     185task           dettool.processexp.run
     186  periods      -poll 0.5
     187  periods      -exec 2.0
     188  periods      -timeout 60
     189
     190  task.exec
     191    queuesize DetrendProcessExposures -var N
     192    if ($N == 0) break
     193    if ($network == 0) break
     194   
     195    # look for new images on the internal DetrendProcessExposures queue
     196    # caution with these 'if' statements: syntax errors
     197    # will make the task fail without given a good status
     198    queuepop DetrendProcessExposures -var line -key 0 NEW
     199    if ("$line" == "NULL") break
     200
     201    strpop line state
     202    queuepush DetrendProcessExposures -replace -key 1:3:5 "RUN $line"
     203
     204    ## the DetrendProcessExposures queue contains:
     205    # STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI
     206    ## NOTE: the queue has the STATE prepended
     207    list word -split $line
     208    $DET_ID   = $word:0
     209    $TYPE     = $word:1
     210    $EXP_KEY  = $word:2
     211
     212    stdout $EXP_KEY.detproc.$CLASS_ID.log
     213    stderr $EXP_KEY.detproc.$CLASS_ID.log
     214
     215    # specify choice of remote host:
     216    if ($parallel)
     217      host anyhost
     218    else
     219      host local
     220    end
     221
     222    # create example job options as a demonstration
     223    options "$line"
     224    echo command detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_KEY --det_type bias
     225    command detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_KEY --det_type bias
     226  end
     227
     228  # success
     229  task.exit 0
     230    queuepush DetrendProcessExposures -replace -key 1:3:5 "DONE $options:0"
     231  end
     232
     233  # default exit status
     234  task.exit    default
     235    queuepush DetrendProcessExposures -replace -key 1:3:5 "FAIL $options:0"
     236  end
     237
     238  # operation times out?
     239  task.exit    timeout
     240    queuepush DetrendProcessExposures -replace -key 1:3:5 "TIMEOUT $options:0"
     241  end
     242end

  • trunk/ippTasks/detrend.reject.pro

    r9081 r9104  
    1313## STATE DET_ID ITER TYPE
    1414
    15 queueinit DetrendRejectExp
     15queueinit DetrendRejectExposure
    1616if ($?network == 0)
    1717  $network = 1
    … …  
    2222
    2323# select images ready for copy
    24 # new entries are added to queue DetrendRejectExp
     24# new entries are added to queue DetrendRejectExposure
    2525# compare the new list with the ones already selected
    2626task           dettool.reject.load
    … …  
    4848      $ITER     = $word:1
    4949      $DET_TYPE = $word:2
    50       queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE"
     50      queuepush DetrendRejectExposure -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE"
    5151    end
    5252
    5353    # delete existing entries which are DONE
    54     queuedrop DetrendRejectExp -key 0 DONE
     54    queuedrop DetrendRejectExposure -key 0 DONE
    5555  end
    5656
    5757  # locked list
    58   task.exit    1
    59     echo       "new.images: exec failure"
    60   end
    61 
    62   # default exit status
    6358  task.exit    default
    64     echo       "new.images: unknown exit status: $EXIT"
     59    echo       "detrend.reject: failure"
    6560  end
    6661
    6762  # operation times out?
    6863  task.exit    timeout
    69     echo       "new.images: timeout"
     64    echo       "detrend.reject: timeout"
    7065  end
    7166end
    7267
    7368# copy new images, sending job to desired host
    74 task           dettool.reject.process
     69task           dettool.reject.run
    7570  periods      -poll 0.5
    7671  periods      -exec 1.0
    … …  
    7873
    7974  task.exec
    80     queuesize DetrendRejectExp -var N
     75    queuesize DetrendRejectExposure -var N
    8176    if ($N == 0) break
    8277    if ($network == 0) break
    8378   
    84     # look for new images on the internal DetrendRejectExp queue
     79    # look for new images on the internal DetrendRejectExposure queue
    8580    # caution with these 'if' statements: syntax errors
    8681    # will make the task fail without given a good status
    87     queuepop DetrendRejectExp -var line -key 0 NEW
     82    queuepop DetrendRejectExposure -var line -key 0 NEW
    8883    if ("$line" == "NULL") break
    8984
    9085    strpop line state
    91     queuepush DetrendRejectExp -replace -key 1:2 "RUN $line"
     86    queuepush DetrendRejectExposure -replace -key 1:2 "RUN $line"
    9287
    93     ## the DetrendRejectExp queue contains:
     88    ## the DetrendRejectExposure queue contains:
    9489    ## STATE DET_ID ITER TYPE
    9590    ## NOTE: the queue has the STATE prepended
    … …  
    118113  # success
    119114  task.exit 0
    120     queuepush DetrendRejectExp -replace -key 1:2 "DONE $options:0"
     115    queuepush DetrendRejectExposure -replace -key 1:2 "DONE $options:0"
    121116  end
    122117
    123118  # default exit status
    124119  task.exit    default
    125     queuepush DetrendRejectExp -replace -key 1:2 "FAIL $options:0"
     120    queuepush DetrendRejectExposure -replace -key 1:2 "FAIL $options:0"
    126121  end
    127122
    128123  # operation times out?
    129124  task.exit    timeout
    130     queuepush DetrendRejectExp -replace -key 1:2 "TIMEOUT $options:0"
     125    queuepush DetrendRejectExposure -replace -key 1:2 "TIMEOUT $options:0"
    131126  end
    132127end

  • trunk/ippTasks/detrend.resid.pro

    r9081 r9104  
    1515
    1616queueinit DetrendResidImfiles
    17 queueinit DetrendResidExp
     17queueinit DetrendResidExposures
    1818if ($?network == 0)
    1919  $network = 1
    … …  
    6565
    6666  # locked list
    67   task.exit    1
    68     echo       "new.images: exec failure"
    69   end
    70 
    71   # default exit status
    72   task.exit    default
    73     echo       "new.images: unknown exit status: $EXIT"
    74   end
    75 
    76   # operation times out?
    77   task.exit    timeout
    78     echo       "new.images: timeout"
     67  # default exit status
     68  task.exit    default
     69    echo       "detrend.resid: failure"
     70  end
     71
     72  # operation times out?
     73  task.exit    timeout
     74    echo       "detrend.resid: timeout"
    7975  end
    8076end
    8177
    8278# copy new images, sending job to desired host
    83 task           dettool.resid.process
     79task           dettool.resid.run
    8480  periods      -poll 0.5
    8581  periods      -exec 5.0
    … …  
    110106    $CLASS_ID = $word:4
    111107    $URI_IN   = $word:5
     108    $CAMERA   = $word:6
    112109
    113110    stdout $EXP_KEY.detresid.$DET_ID.$ITER.$CLASS_ID.log
    … …  
    124121    options "$line"
    125122    # detrend_create_resid.pl DET_ID ITER EXP_KEY CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_KEY ???)
    126     $URL_DET = isp.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
    127     # $URL_DET = $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
     123    # $URL_DET = isp.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
     124    $URL_DET = $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
    128125    echo command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN
    129126    command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN
    … …  
    147144
    148145# select images ready for copy
    149 # new entries are added to queue DetrendResidExp
     146# new entries are added to queue DetrendResidExposures
    150147# compare the new list with the ones already selected
    151148task           dettool.residexp.load
    … …  
    177174      $EXP_KEY  = $word:3
    178175      $INCLUDE  = $word:4
    179       queuepush DetrendResidExp -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_KEY $INCLUDE"
     176      queuepush DetrendResidExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_KEY $INCLUDE"
    180177    end
    181178
    182179    # delete existing entries which are DONE
    183     queuedrop DetrendResidExp -key 0 DONE
     180    queuedrop DetrendResidExposures -key 0 DONE
    184181  end
    185182
    186183  # locked list
    187   task.exit    1
    188     echo       "new.images: exec failure"
    189   end
    190 
    191   # default exit status
    192   task.exit    default
    193     echo       "new.images: unknown exit status: $EXIT"
    194   end
    195 
    196   # operation times out?
    197   task.exit    timeout
    198     echo       "new.images: timeout"
     184  # default exit status
     185  task.exit    default
     186    echo       "detrend.residexp: failure"
     187  end
     188
     189  # operation times out?
     190  task.exit    timeout
     191    echo       "detrend.residexp: timeout"
    199192  end
    200193end
    201194
    202195# copy new images, sending job to desired host
    203 task           dettool.residexp.process
     196task           dettool.residexp.run
    204197  periods      -poll 0.5
    205198  periods      -exec 5.0
    … …  
    207200
    208201  task.exec
    209     queuesize DetrendResidExp -var N
     202    queuesize DetrendResidExposures -var N
    210203    if ($N == 0) break
    211204    if ($network == 0) break
    212205   
    213     # look for new images on the internal DetrendResidExp queue
     206    # look for new images on the internal DetrendResidExposures queue
    214207    # caution with these 'if' statements: syntax errors
    215208    # will make the task fail without given a good status
    216     queuepop DetrendResidExp -var line -key 0 NEW
     209    queuepop DetrendResidExposures -var line -key 0 NEW
    217210    if ("$line" == "NULL") break
    218211
    219212    strpop line state
    220     queuepush DetrendResidExp -replace -key 1:2:4 "RUN $line"
    221 
    222     ## the DetrendResidExp queue contains:
     213    queuepush DetrendResidExposures -replace -key 1:2:4 "RUN $line"
     214
     215    ## the DetrendResidExposures queue contains:
    223216    ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN
    224217    ## NOTE: the queue has the STATE prepended
    … …  
    248241  # success
    249242  task.exit 0
    250     queuepush DetrendResidExp -replace -key 1:2:4 "DONE $options:0"
    251   end
    252 
    253   # default exit status
    254   task.exit    default
    255     queuepush DetrendResidExp -replace -key 1:2:4 "FAIL $options:0"
    256   end
    257 
    258   # operation times out?
    259   task.exit    timeout
    260     queuepush DetrendResidExp -replace -key 1:2:4 "TIMEOUT $options:0"
    261   end
    262 end
     243    queuepush DetrendResidExposures -replace -key 1:2:4 "DONE $options:0"
     244  end
     245
     246  # default exit status
     247  task.exit    default
     248    queuepush DetrendResidExposures -replace -key 1:2:4 "FAIL $options:0"
     249  end
     250
     251  # operation times out?
     252  task.exit    timeout
     253    queuepush DetrendResidExposures -replace -key 1:2:4 "TIMEOUT $options:0"
     254  end
     255end

  • trunk/ippTasks/detrend.stack.pro

    r9081 r9104  
    1515## STATE DET_ID ITER TYPE CLASS_ID CAMERA
    1616
    17 queueinit DetrendClassIDtoStack
     17queueinit DetrendStackClass
    1818if ($?network == 0)
    1919  $network = 1
    … …  
    2424
    2525# select images ready for copy
    26 # new entries are added to queue DetrendClassIDtoStack
     26# new entries are added to queue DetrendStackClass
    2727# compare the new list with the ones already selected
    2828task           dettool.stack.load
    … …  
    5454      $CLASS_ID = $word:3
    5555      $CAMERA   = $word:4
    56       echo queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA"
    57       queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA"
     56      echo queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA"
     57      queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA"
    5858    end
    5959
    6060    # delete existing entries which are DONE
    61     queuedrop DetrendClassIDtoStack -key 0 DONE
     61    queuedrop DetrendStackClass -key 0 DONE
    6262  end
    6363
    6464  # locked list
    65   task.exit    1
    66     echo       "new.images: exec failure"
    67   end
    68 
    69   # default exit status
    7065  task.exit    default
    71     echo       "new.images: unknown exit status: $EXIT"
     66    echo       "detrend.stack: failure"
    7267  end
    7368
    7469  # operation times out?
    7570  task.exit    timeout
    76     echo       "new.images: timeout"
     71    echo       "detrend.stack: timeout"
    7772  end
    7873end
    7974
    8075# copy new images, sending job to desired host
    81 task           dettool.stack.process
     76task           dettool.stack.run
    8277  periods      -poll 0.5
    8378  periods      -exec 1.0
    … …  
    8580
    8681  task.exec
    87     queuesize DetrendClassIDtoStack -var N
     82    queuesize DetrendStackClass -var N
    8883   
    8984    if ($N == 0) break
    9085    if ($network == 0) break
    9186   
    92     # look for new images on the internal DetrendClassIDtoStack queue
     87    # look for new images on the internal DetrendStackClass queue
    9388    # caution with these 'if' statements: syntax errors
    9489    # will make the task fail without given a good status
    95     queuepop DetrendClassIDtoStack -var line -key 0 NEW
     90    queuepop DetrendStackClass -var line -key 0 NEW
    9691    if ("$line" == "NULL") break
    9792
    9893    strpop line state
    99     queuepush DetrendClassIDtoStack -replace -key 1:2:4 "RUN $line"
     94    queuepush DetrendStackClass -replace -key 1:2:4 "RUN $line"
    10095
    101     # the DetrendClassIDtoStack queue contains:
     96    # the DetrendStackClass queue contains:
    10297    # STATE DET_ID ITER TYPE CLASS_ID
    10398    ## NOTE: the queue has the STATE prepended
    … …  
    108103    $TYPE     = $word:2
    109104    $CLASS_ID = $word:3
    110     $CAMERA   = $word:3
     105    $CAMERA   = $word:4
    111106
    112107    stdout $CAMERA.$TYPE.$DET_ID.$ITER.$CLASS_ID.log
    … …  
    129124  # success
    130125  task.exit 0
    131     queuepush DetrendClassIDtoStack -replace -key 1:2:4 "DONE $options:0"
     126    queuepush DetrendStackClass -replace -key 1:2:4 "DONE $options:0"
    132127  end
    133128
    134129  # default exit status
    135130  task.exit    default
    136     queuepush DetrendClassIDtoStack -replace -key 1:2:4 "FAIL $options:0"
     131    queuepush DetrendStackClass -replace -key 1:2:4 "FAIL $options:0"
    137132  end
    138133
    139134  # operation times out?
    140135  task.exit    timeout
    141     queuepush DetrendClassIDtoStack -replace -key 1:2:4 "TIMEOUT $options:0"
     136    queuepush DetrendStackClass -replace -key 1:2:4 "TIMEOUT $options:0"
    142137  end
    143138end

  • trunk/ippTasks/pantasks.pro

    r9081 r9104  
    4040end
    4141
     42macro step2.flat
     43  exec dettool -definebyquery -det_type flat -exp_type flat -camera isp
     44  input $scripts/detrend.process.pro
     45  run
     46end
     47
    4248macro step2.both
    4349  exec dettool -definebyquery -det_type bias -exp_type bias -camera isp
    4450  exec dettool -definebyquery -det_type dark -exp_type dark -camera isp
     51  exec dettool -definebyquery -det_type flat -exp_type flat -camera isp
    4552  input $scripts/detrend.process.pro
    4653  run
    … …  
    4956macro step3
    5057  input $scripts/detrend.stack.pro
     58  input $scripts/detrend.norm.pro
    5159  input $scripts/detrend.resid.pro
    5260  input $scripts/detrend.reject.pro

  • trunk/ippTasks/phase0.pro

    r9081 r9104  
    22## this file contains the tasks for running the phase 0 stage
    33
    4 ## these tasks use the queue Phase0PendingImfiles
    5 ## the Phase0PendingImfiles queue contains:
     4## these tasks use the queue Phase0Imfiles
     5## the Phase0Imfiles queue contains:
    66## STATE EXP_KEY CLASS CLASS_ID URI-FITS
    77## queue keys are counted from 0
    88
    9 queueinit Phase0PendingImfiles
    10 queueinit Phase0PendingExposures
     9queueinit Phase0Imfiles
     10queueinit Phase0Exposures
    1111if ($?network == 0)
    1212  $network = 1
    … …  
    1717
    1818# select images ready for phase0 analysis
    19 # new entries are added to queue Phase0PendingImfiles
     19# new entries are added to queue Phase0Imfiles
    2020# compare the new list with the ones already selected
    2121task           phase0.imfile.load
    … …  
    3636    # save stdout and stderr
    3737
    38     # compare output with Phase0PendingImfiles queue
     38    # compare output with Phase0Imfiles queue
    3939    # only add entries which don't exist in queue
    4040    queuesize stdout -var Nqueue
    4141    for i 0 $Nqueue
    4242      queuepop stdout -var line
    43       queuepush Phase0PendingImfiles -uniq -key 1 "NEW $line"
     43      queuepush Phase0Imfiles -uniq -key 1 "NEW $line"
    4444    end
    4545
    4646    # delete existing entries which are DONE
    47     queuedrop Phase0PendingImfiles -key 0 DONE
     47    queuedrop Phase0Imfiles -key 0 DONE
    4848  end
    4949
    5050  # locked list
    5151  task.exit    default
    52     echo "*** phase0 pending imfiles: failure"
    53   end
    54 
    55   # operation times out?
    56   task.exit    timeout
    57     echo "*** phase0 pending imfiles: timeout"
     52    echo "phase0 pending imfiles: failure"
     53  end
     54
     55  # operation times out?
     56  task.exit    timeout
     57    echo "phase0 pending imfiles: timeout"
    5858  end
    5959end
    6060
    6161# run the phase0imfile script on pending images
    62 task           phase0.imfile.process
     62task           phase0.imfile.run
    6363  periods      -poll 0.5
    6464  periods      -exec 1
    … …  
    6666
    6767  task.exec
    68     queuesize Phase0PendingImfiles -var N
     68    queuesize Phase0Imfiles -var N
    6969    if ($N == 0) break
    7070    if ($network == 0) break
    … …  
    7373    # caution with these 'if' statements: syntax errors
    7474    # will make the task fail without given a good status
    75     queuepop Phase0PendingImfiles -var line -key 0 NEW
     75    queuepop Phase0Imfiles -var line -key 0 NEW
    7676    if ("$line" == "NULL") break
    7777
    7878    strpop line state
    79     queuepush Phase0PendingImfiles -replace -key 1 "RUN $line"
    80 
    81     ## the Phase0PendingImfiles queue contains (line has popped STATE)
     79    queuepush Phase0Imfiles -replace -key 1 "RUN $line"
     80
     81    ## the Phase0Imfiles queue contains (line has popped STATE)
    8282    ## STATE EXP_KEY CLASS CLASS_ID URI-FITS
    8383    list word -split $line
    … …  
    103103  task.exit 0
    104104    # phase0imfile.pl updates DB tables, here we just update the queue
    105     queuepush Phase0PendingImfiles -replace -key 1 "DONE $options:0"
     105    queuepush Phase0Imfiles -replace -key 1 "DONE $options:0"
    106106  end
    107107
    … …  
    109109  task.exit    default
    110110    echo "failure for phase0imfile.pl: $options:0"
    111     queuepush Phase0PendingImfiles -replace -key 1 "FAIL $options:0"
     111    queuepush Phase0Imfiles -replace -key 1 "FAIL $options:0"
    112112  end
    113113
    … …  
    115115  task.exit    timeout
    116116    echo "timeout for phase0imfile.pl: $options:0"
    117     queuepush Phase0PendingImfiles -replace -key 1 "TIMEOUT $options:0"
    118   end
    119 end
    120 
    121 ## these tasks use the queue Phase0PendingExposures
    122 ## the Phase0PendingExposures queue contains:
     117    queuepush Phase0Imfiles -replace -key 1 "TIMEOUT $options:0"
     118  end
     119end
     120
     121## these tasks use the queue Phase0Exposures
     122## the Phase0Exposures queue contains:
    123123## STATE EXP_KEY CAMERA TELESCOPE TYPE NCLASS
    124124
    125125# select images ready for phase0 analysis
    126 # new entries are added to queue Phase0PendingExposures
     126# new entries are added to queue Phase0Exposures
    127127# compare the new list with the ones already selected
    128128task           phase0.exp.load
    … …  
    141141    local i Nqueue
    142142
    143     # compare output with Phase0PendingExposures queue
     143    # compare output with Phase0Exposures queue
    144144    # only add entries which don't exist in queue
    145145    queuesize stdout -var Nqueue
    146146    for i 0 $Nqueue
    147147      queuepop stdout -var line
    148       queuepush Phase0PendingExposures -uniq -key 1 "NEW $line"
     148      queuepush Phase0Exposures -uniq -key 1 "NEW $line"
    149149    end
    150150
    151151    # delete existing entries which are DONE
    152     queuedrop Phase0PendingExposures -key 0 DONE
     152    queuedrop Phase0Exposures -key 0 DONE
    153153  end
    154154
    … …  
    165165
    166166# run the phase0imfile script on pending images
    167 task           phase0.exp.process
     167task           phase0.exp.run
    168168  periods      -poll 0.5
    169169  periods      -exec 1.0
    … …  
    171171
    172172  task.exec
    173     queuesize Phase0PendingExposures -var N
     173    queuesize Phase0Exposures -var N
    174174    if ($N == 0) break
    175175    if ($network == 0) break
    … …  
    178178    # caution with these 'if' statements: syntax errors
    179179    # will make the task fail without given a good status
    180     queuepop Phase0PendingExposures -var line -key 0 NEW
     180    queuepop Phase0Exposures -var line -key 0 NEW
    181181    if ("$line" == "NULL") break
    182182
    183183    strpop line state
    184     queuepush Phase0PendingExposures -replace -key 1 "RUN $line"
    185 
    186     ## the Phase0PendingExposures queue contains:
     184    queuepush Phase0Exposures -replace -key 1 "RUN $line"
     185
     186    ## the Phase0Exposures queue contains:
    187187    ## STATE EXP_KEY CAMERA TELESCOPE TYPE NCLASS
    188188    list word -split $line
    … …  
    208208  task.exit 0
    209209    # phase0exp.pl updates DB tables, here we just update the queue
    210     queuepush Phase0PendingExposures -replace -key 1 "DONE $options:0"
     210    queuepush Phase0Exposures -replace -key 1 "DONE $options:0"
    211211  end
    212212
    213213  # default exit status
    214214  task.exit    default
    215     queuepush Phase0PendingExposures -replace -key 1 "FAIL $options:0"
    216   end
    217 
    218   # operation times out?
    219   task.exit    timeout
    220     queuepush Phase0PendingExposures -replace -key 1 "TIMEOUT $options:0"
     215    queuepush Phase0Exposures -replace -key 1 "FAIL $options:0"
     216  end
     217
     218  # operation times out?
     219  task.exit    timeout
     220    queuepush Phase0Exposures -replace -key 1 "TIMEOUT $options:0"
    221221  end
    222222end
Note: See TracChangeset for help on using the changeset viewer.