IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 9338


Ignore:
Timestamp:
Oct 5, 2006, 5:20:43 PM (20 years ago)
Author:
eugene
Message:

substantial work to use current db tables

Location:
trunk/ippTasks
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/detrend.norm.pro

    r9104 r9338  
    1414if ($?parallel == 0)
    1515  $parallel = 0
     16end
     17
     18# select images ready for copy
     19# new entries are added to queue DetrendNormStat
     20# compare the new list with the ones already selected
     21task           dettool.normstat.load
     22  command      dettool -tonormalizedstat -simple
     23  host         local
     24
     25  periods      -poll 1
     26  periods      -exec 5
     27  periods      -timeout 5
     28
     29  stdout dettool.normstat.log
     30  stderr dettool.normstat.log
     31
     32  # success
     33  task.exit    0
     34    # XXX is it necessary for these to be local?
     35    # drop after this task macro is done?
     36    local i Nqueue
     37
     38    # compare output with newImage queue
     39    # only add entries which don't exist in queue
     40    queuesize stdout -var Nqueue
     41    for i 0 $Nqueue
     42      queuepop stdout -var line
     43      list word -split $line
     44      $DET_ID   = $word:0
     45      $DET_TYPE = $word:1
     46      $ITER     = $word:2
     47      $CAMERA   = $word:3
     48      queuepush DetrendNormStat -uniq -key 1:3 "NEW $DET_ID $DET_TYPE $ITER $CAMERA"
     49    end
     50
     51    # delete existing entries which are DONE
     52    queuedrop DetrendNormStat -key 0 DONE
     53  end
     54
     55  # locked list
     56  # default exit status
     57  task.exit    default
     58    echo       "detrend.norm: failure"
     59  end
     60
     61  # operation times out?
     62  task.exit    timeout
     63    echo       "detrend.norm: timeout"
     64  end
     65end
     66
     67# copy new images, sending job to desired host
     68task           dettool.normstat.process
     69  periods      -poll 0.5
     70  periods      -exec 5.0
     71  periods      -timeout 30
     72
     73  task.exec
     74    queuesize DetrendNormStat -var N
     75   
     76    if ($N == 0) break
     77    if ($network == 0) break
     78   
     79    # look for new images on the internal DetrendNormStat queue
     80    # caution with these 'if' statements: syntax errors
     81    # will make the task fail without given a good status
     82    queuepop DetrendNormStat -var line -key 0 NEW
     83    if ("$line" == "NULL") break
     84
     85    strpop line state
     86    queuepush DetrendNormStat -replace -key 1:3 "RUN $line"
     87
     88    # the DetrendNormStat queue contains:
     89    # STATE DET_ID ITER
     90    list word -split $line
     91    $DET_ID   = $word:0
     92    $DET_TYPE = $word:1
     93    $ITER     = $word:2
     94    $CAMERA   = $word:3
     95
     96    # XXX add $WORKDIR/$LOG_DIR
     97    stdout $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
     98    stderr $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
     99
     100    # specify choice of remote host:
     101    if ($parallel)
     102      host anyhost
     103    else
     104      host local
     105    end
     106
     107    # create example job options as a demonstration
     108    options "$line"
     109    echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE
     110    command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE
     111  end
     112
     113  # success
     114  task.exit 0
     115    queuepush DetrendNormStat -replace -key 1:3 "DONE $options:0"
     116  end
     117
     118  # default exit status
     119  task.exit    default
     120    queuepush DetrendNormStat -replace -key 1:3 "FAIL $options:0"
     121  end
     122
     123  # operation times out?
     124  task.exit    timeout
     125    queuepush DetrendNormStat -replace -key 1:3 "TIMEOUT $options:0"
     126  end
    16127end
    17128
    … …  
    41152    for i 0 $Nqueue
    42153      queuepop stdout -var line
    43       queuepush DetrendNormalize -uniq -key 1:2 "NEW $line"
     154      list word -split $line
     155      $DET_TYPE = $word:0
     156      $CAMERA   = $word:1
     157      $URI      = $word:2
     158      $DET_ID   = $word:3
     159      $ITER     = $word:4
     160      $CLASS_ID = $word:5
     161      $NORM     = $word:6
     162      queuepush DetrendNormalize -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM"
    44163    end
    45164
    … …  
    79198
    80199    strpop line state
    81     queuepush DetrendNormalize -replace -key 1:2 "RUN $line"
     200    queuepush DetrendNormalize -replace -key 4:5 "RUN $line"
    82201
    83202    # the DetrendNormalize queue contains:
    84203    # STATE DET_ID ITER
    85204    list word -split $line
    86     $DET_ID   = $word:0
    87     $ITER     = $word:1
    88 
    89     # XXX we need to get CAMERA and TYPE from this query
    90     # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log
    91     # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log
    92     stdout isp.flat.$DET_ID.$ITER.log
    93     stderr isp.flat.$DET_ID.$ITER.log
     205    $DET_TYPE = $word:0
     206    $CAMERA   = $word:1
     207    $URI      = $word:2
     208    $DET_ID   = $word:3
     209    $ITER     = $word:4
     210    $CLASS_ID = $word:5
     211    $NORM     = $word:6
     212
     213    $LOG_DIR = `dirname $URI`
     214
     215    stdout $WORKDIR/$LOG_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
     216    stderr $WORKDIR/$LOG_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
    94217
    95218    # specify choice of remote host:
    … …  
    102225    # create example job options as a demonstration
    103226    options "$line"
    104     echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER
    105     command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER
     227    echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
     228    command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
    106229  end
    107230
    108231  # success
    109232  task.exit 0
    110     queuepush DetrendNormalize -replace -key 1:2 "DONE $options:0"
    111   end
    112 
    113   # default exit status
    114   task.exit    default
    115     queuepush DetrendNormalize -replace -key 1:2 "FAIL $options:0"
    116   end
    117 
    118   # operation times out?
    119   task.exit    timeout
    120     queuepush DetrendNormalize -replace -key 1:2 "TIMEOUT $options:0"
    121   end
    122 end
    123 
    124 # select images ready for copy
    125 # new entries are added to queue DetrendNormStat
    126 # compare the new list with the ones already selected
    127 task           dettool.normstat.load
    128   command      dettool -normstat -simple
    129   host         local
    130 
    131   periods      -poll 1
    132   periods      -exec 5
    133   periods      -timeout 5
    134 
    135   stdout dettool.normstat.log
    136   stderr dettool.normstat.log
    137 
    138   # success
    139   task.exit    0
    140     # XXX is it necessary for these to be local?
    141     # drop after this task macro is done?
    142     local i Nqueue
    143 
    144     # compare output with newImage queue
    145     # only add entries which don't exist in queue
    146     queuesize stdout -var Nqueue
    147     for i 0 $Nqueue
    148       queuepop stdout -var line
    149       queuepush DetrendNormStat -uniq -key 1:2 "NEW $line"
    150     end
    151 
    152     # delete existing entries which are DONE
    153     queuedrop DetrendNormStat -key 0 DONE
    154   end
    155 
    156   # locked list
    157   # default exit status
    158   task.exit    default
    159     echo       "detrend.norm: failure"
    160   end
    161 
    162   # operation times out?
    163   task.exit    timeout
    164     echo       "detrend.norm: timeout"
    165   end
    166 end
    167 
    168 # copy new images, sending job to desired host
    169 task           dettool.normstat.process
    170   periods      -poll 0.5
    171   periods      -exec 5.0
    172   periods      -timeout 30
    173 
    174   task.exec
    175     queuesize DetrendNormStat -var N
    176    
    177     if ($N == 0) break
    178     if ($network == 0) break
    179    
    180     # look for new images on the internal DetrendNormStat queue
    181     # caution with these 'if' statements: syntax errors
    182     # will make the task fail without given a good status
    183     queuepop DetrendNormStat -var line -key 0 NEW
    184     if ("$line" == "NULL") break
    185 
    186     strpop line state
    187     queuepush DetrendNormStat -replace -key 1:2 "RUN $line"
    188 
    189     # the DetrendNormStat queue contains:
    190     # STATE DET_ID ITER
    191     list word -split $line
    192     $DET_ID   = $word:0
    193     $ITER     = $word:1
    194 
    195     # XXX we need to get CAMERA and TYPE from this query
    196     # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log
    197     # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log
    198     stdout isp.flat.$DET_ID.$ITER.log
    199     stderr isp.flat.$DET_ID.$ITER.log
    200 
    201     # specify choice of remote host:
    202     if ($parallel)
    203       host anyhost
    204     else
    205       host local
    206     end
    207 
    208     # create example job options as a demonstration
    209     options "$line"
    210     echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
    211     command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
    212   end
    213 
    214   # success
    215   task.exit 0
    216     queuepush DetrendNormStat -replace -key 1:2 "DONE $options:0"
    217   end
    218 
    219   # default exit status
    220   task.exit    default
    221     queuepush DetrendNormStat -replace -key 1:2 "FAIL $options:0"
    222   end
    223 
    224   # operation times out?
    225   task.exit    timeout
    226     queuepush DetrendNormStat -replace -key 1:2 "TIMEOUT $options:0"
     233    queuepush DetrendNormalize -replace -key 4:5 "DONE $options:0"
     234  end
     235
     236  # default exit status
     237  task.exit    default
     238    queuepush DetrendNormalize -replace -key 4:5 "FAIL $options:0"
     239  end
     240
     241  # operation times out?
     242  task.exit    timeout
     243    queuepush DetrendNormalize -replace -key 4:5 "TIMEOUT $options:0"
    227244  end
    228245end
    … …  
    253270    for i 0 $Nqueue
    254271      queuepop stdout -var line
    255       queuepush DetrendNormalizeExposures -uniq -key 1:2 "NEW $line"
     272      list word -split $line
     273      $DET_ID   = $word:0
     274      $ITER     = $word:1
     275      $DET_TYPE = $word:2
     276      $CAMERA   = $word:3
     277      #$TELESCOPE= $word:4
     278      #$EXP_TYPE = $word:5
     279      #$IMFILES  = $word:6
     280      queuepush DetrendNormalizeExposures -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA"
    256281    end
    257282
    … …  
    298323    $DET_ID   = $word:0
    299324    $ITER     = $word:1
    300 
    301     # XXX we need to get CAMERA and TYPE from this query
    302     # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log
    303     # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log
    304     stdout isp.flat.$DET_ID.$ITER.log
    305     stderr isp.flat.$DET_ID.$ITER.log
     325    $DET_TYPE = $word:2
     326    $CAMERA   = $word:3
     327    #$TELESCOPE= $word:4
     328    #$EXP_TYPE = $word:5
     329    #$IMFILES  = $word:6
     330
     331    # XXX add $WORKDIR/$LOG_DIR
     332    stdout $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
     333    stderr $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
    306334
    307335    # specify choice of remote host:
    … …  
    314342    # create example job options as a demonstration
    315343    options "$line"
    316     echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
    317     command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
     344    echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE
     345    command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE
    318346  end
    319347

  • trunk/ippTasks/detrend.process.pro

    r9104 r9338  
    1111## these tasks use the queue DetrendProcessImfiles
    1212## the DetrendProcessImfiles queue contains:
    13 ## STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI
     13## STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI
    1414
    1515queueinit DetrendProcessImfiles
    … …  
    2626# compare the new list with the ones already selected
    2727task           dettool.process.load
    28   command      dettool -raw -simple
     28  command      dettool -toprocess -simple
    2929  host         local
    3030
    … …  
    4949      list word -split $line
    5050      $DET_ID   = $word:0
    51       $TYPE    = $word:1
    52       $EXP_KEY  = $word:2
     51      $DET_TYPE = $word:1
     52      $EXP_TAG  = $word:2
    5353      $CLASS    = $word:3
    5454      $CLASS_ID = $word:4
    5555      $URI      = $word:5
    56       queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $TYPE $EXP_KEY $CLASS $CLASS_ID $URI"
     56      queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $DET_TYPE $EXP_TAG $CLASS $CLASS_ID $URI"
    5757    end
    5858
    … …  
    9393
    9494    ## the DetrendProcessImfiles queue contains:
    95     # STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI
     95    # STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI
    9696    ## NOTE: the queue has the STATE prepended
    9797    list word -split $line
    9898    $DET_ID   = $word:0
    99     $TYPE    = $word:1
    100     $EXP_KEY  = $word:2
     99    $DET_TYPE = $word:1
     100    $EXP_TAG  = $word:2
    101101    $CLASS    = $word:3
    102102    $CLASS_ID = $word:4
    103103    $URI      = $word:5
    104104
    105     stdout $EXP_KEY.detproc.$CLASS_ID.log
    106     stderr $EXP_KEY.detproc.$CLASS_ID.log
     105    $LOG_DIR = `dirname $URI`
     106    stdout $WORKDIR/$LOG_DIR/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log
     107    stderr $WORKDIR/$LOG_DIR/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log
    107108
    108109    # specify choice of remote host:
    … …  
    115116    # create example job options as a demonstration
    116117    options "$line"
    117     echo command detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI
    118     command detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI
     118    echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI
     119    command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI
    119120  end
    120121
    121122  # success
    122123  task.exit 0
     124    echo "success detrend_process_imfile.pl $options:0"
    123125    queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0"
    124126  end
    … …  
    126128  # default exit status
    127129  task.exit    default
     130    echo "failure detrend_process_imfile.pl $options:0"
    128131    queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0"
    129132  end
    … …  
    131134  # operation times out?
    132135  task.exit    timeout
     136    echo "timeout detrend_process_imfile.pl $options:0"
    133137    queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0"
    134138  end
    … …  
    139143# compare the new list with the ones already selected
    140144task           dettool.processexp.load
    141   command      dettool -toprocessdexp -simple
     145  command      dettool -toprocessedexp -simple
    142146  host         local
    143147
    … …  
    162166      list word -split $line
    163167      $DET_ID   = $word:0
    164       $TYPE     = $word:1
    165       $EXP_KEY  = $word:2
    166       queuepush DetrendProcessExposures -uniq -key 1:3 "NEW $DET_ID $TYPE $EXP_KEY"
     168      $ITER     = $word:1
     169      $DET_TYPE = $word:2
     170      $EXP_TAG  = $word:3
     171      queuepush DetrendProcessExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_TAG"
    167172    end
    168173
    … …  
    200205
    201206    strpop line state
    202     queuepush DetrendProcessExposures -replace -key 1:3:5 "RUN $line"
    203 
    204     ## the DetrendProcessExposures queue contains:
    205     # STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI
    206     ## NOTE: the queue has the STATE prepended
     207    queuepush DetrendProcessExposures -replace -key 1:2:4 "RUN $line"
     208
     209
    207210    list word -split $line
    208211    $DET_ID   = $word:0
    209     $TYPE     = $word:1
    210     $EXP_KEY  = $word:2
    211 
    212     stdout $EXP_KEY.detproc.$CLASS_ID.log
    213     stderr $EXP_KEY.detproc.$CLASS_ID.log
     212    $ITER     = $word:1
     213    $DET_TYPE = $word:2
     214    $EXP_TAG  = $word:3
     215
     216    # XXX need the classID in dettool -toprocessedexp -simple output
     217    $CLASS_ID = isp
     218    stdout $EXP_TAG.detproc.$DET_ID.log
     219    stderr $EXP_TAG.detproc.$DET_ID.log
    214220
    215221    # specify choice of remote host:
    … …  
    222228    # create example job options as a demonstration
    223229    options "$line"
    224     echo command detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_KEY --det_type bias
    225     command detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_KEY --det_type bias
     230    echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG
     231    command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG
    226232  end
    227233
    228234  # success
    229235  task.exit 0
    230     queuepush DetrendProcessExposures -replace -key 1:3:5 "DONE $options:0"
     236    queuepush DetrendProcessExposures -replace -key 1:2:4 "DONE $options:0"
    231237  end
    232238
    233239  # default exit status
    234240  task.exit    default
    235     queuepush DetrendProcessExposures -replace -key 1:3:5 "FAIL $options:0"
    236   end
    237 
    238   # operation times out?
    239   task.exit    timeout
    240     queuepush DetrendProcessExposures -replace -key 1:3:5 "TIMEOUT $options:0"
    241   end
    242 end
     241    queuepush DetrendProcessExposures -replace -key 1:2:4 "FAIL $options:0"
     242  end
     243
     244  # operation times out?
     245  task.exit    timeout
     246    queuepush DetrendProcessExposures -replace -key 1:2:4 "TIMEOUT $options:0"
     247  end
     248end

  • trunk/ippTasks/detrend.reject.pro

    r9104 r9338  
    4848      $ITER     = $word:1
    4949      $DET_TYPE = $word:2
    50       queuepush DetrendRejectExposure -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE"
     50      $CAMERA   = $word:3
     51      queuepush DetrendRejectExposure -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA"
    5152    end
    5253
    … …  
    9394    $ITER     = $word:1
    9495    $DET_TYPE = $word:2
     96    $CAMERA   = $word:3
    9597
    9698    # I'd like to add CAMERA to the log file...
    97     stdout isp.$DET_TYPE.$DET_ID.$ITER.log
    98     stderr isp.$DET_TYPE.$DET_ID.$ITER.log
     99    stdout $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
     100    stderr $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
    99101
    100102    # specify choice of remote host:

  • trunk/ippTasks/detrend.resid.pro

    r9104 r9338  
    44# dettool -toresid -simple
    55# 0    bias 1      850131b mc       o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582
    6 # ITER TYPE DET_ID EXP_KEY CLASS_ID URI-IN           
     6# ITER TYPE DET_ID EXP_TAG CLASS_ID URI-IN           
    77
    88# detrend_create_resid.pl 1      0    850131b mc       bias bias_mc_1_0.fit o_850131b.mc.fit 850131b
    9 # detrend_create_resid.pl DET_ID ITER EXP_KEY CLASS_ID TYPE (URI-DET)       (URL-IN)         (EXP_KEY)
     9# detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET)       (URL-IN)         (EXP_TAG)
    1010# URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit
    1111
    1212## these tasks use the queue DetrendResidImfiles
    1313## the DetrendResidImfiles queue contains:
    14 ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN
     14## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN
    1515
    1616queueinit DetrendResidImfiles
    … …  
    5252      $DET_TYPE = $word:1
    5353      $DET_ID   = $word:2
    54       $EXP_KEY  = $word:3
     54      $EXP_TAG  = $word:3
    5555      $CLASS_ID = $word:4
    5656      $URI_IN   = $word:5
    57       ## XXX need to include the detID and iter in the key (if we have two running at the same time...)
     57      $CAMERA   = $word:13
     58      ## must include the detID and iter in the key (if we have two running at the same time...)
    5859      ## Also needed to be in the output file (or path)
    59       queuepush DetrendResidImfiles -uniq -key 1:3:4:5 "NEW $ITER $DET_TYPE $DET_ID $EXP_KEY $CLASS_ID $URI_IN"
     60      queuepush DetrendResidImfiles -uniq -key 1:3:4:5 "NEW $ITER $DET_TYPE $DET_ID $EXP_TAG $CLASS_ID $URI_IN $CAMERA"
    6061    end
    6162
    … …  
    9798
    9899    ## the DetrendResidImfiles queue contains:
    99     ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN
     100    ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN
    100101    ## NOTE: the queue has the STATE prepended
    101102    list word -split $line
    … …  
    103104    $DET_TYPE = $word:1
    104105    $DET_ID   = $word:2
    105     $EXP_KEY  = $word:3
     106    $EXP_TAG  = $word:3
    106107    $CLASS_ID = $word:4
    107108    $URI_IN   = $word:5
    108109    $CAMERA   = $word:6
    109110
    110     stdout $EXP_KEY.detresid.$DET_ID.$ITER.$CLASS_ID.log
    111     stderr $EXP_KEY.detresid.$DET_ID.$ITER.$CLASS_ID.log
     111    $DET_DIR = `dirname $URI_IN`
     112    $LOG_DIR = $WORKDIR/$DET_DIR
     113
     114    stdout $LOG_DIR/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
     115    stderr $LOG_DIR/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
    112116
    113117    # specify choice of remote host:
    … …  
    120124    # create example job options as a demonstration
    121125    options "$line"
    122     # detrend_create_resid.pl DET_ID ITER EXP_KEY CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_KEY ???)
    123     # $URL_DET = isp.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
    124     $URL_DET = $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
    125     echo command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN
    126     command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN
     126    # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???)
     127    # need to parse the subdir path for the detrend URL
     128    $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
     129    echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URL --input_uri $URI_IN
     130    command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URL --input_uri $URI_IN
    127131  end
    128132
    … …  
    172176      $ITER     = $word:1
    173177      $DET_TYPE = $word:2
    174       $EXP_KEY  = $word:3
     178      $EXP_TAG  = $word:3
    175179      $INCLUDE  = $word:4
    176       queuepush DetrendResidExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_KEY $INCLUDE"
     180      queuepush DetrendResidExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_TAG $INCLUDE"
    177181    end
    178182
    … …  
    214218
    215219    ## the DetrendResidExposures queue contains:
    216     ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN
     220    ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN
    217221    ## NOTE: the queue has the STATE prepended
    218222    list word -split $line
    … …  
    220224    $ITER     = $word:1
    221225    $DET_TYPE = $word:2
    222     $EXP_KEY  = $word:3
     226    $EXP_TAG  = $word:3
    223227    $INCLUDE  = $word:4
    224228
    225     stdout $EXP_KEY.detresid.$DET_ID.$ITER.log
    226     stderr $EXP_KEY.detresid.$DET_ID.$ITER.log
     229    # XXX add $WORKDIR/$LOG_DIR
     230    stdout $EXP_TAG.detresid.$DET_ID.$ITER.log
     231    stderr $EXP_TAG.detresid.$DET_ID.$ITER.log
    227232
    228233    # specify choice of remote host:
    … …  
    233238    end
    234239
     240    ### XXX artificially reject a few randomly
     241    if ($ITER == 0)
     242      $drop = (rnd(0) < 0.33)
     243    else
     244      $drop = ("$INCLUDE" == "F")
     245    end
     246
    235247    # create example job options as a demonstration
    236248    options "$line"
    237     echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --det_type $DET_TYPE
    238     command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --det_type $DET_TYPE
     249    if ($drop)
     250      echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --reject
     251      command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --reject
     252    else
     253      echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE
     254      command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE
     255    end
    239256  end
    240257

  • trunk/ippTasks/detrend.stack.pro

    r9104 r9338  
    3939  # success
    4040  task.exit    0
    41     # XXX is it necessary for these to be local?
    42     # drop after this task macro is done?
    4341    local i Nqueue
    4442
    … …  
    5149      $DET_ID   = $word:0
    5250      $ITER     = $word:1
    53       $TYPE    = $word:2
     51      $DET_TYPE = $word:2
    5452      $CLASS_ID = $word:3
    5553      $CAMERA   = $word:4
    56       echo queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA"
    57       queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA"
     54      queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $CLASS_ID $CAMERA"
    5855    end
    5956
    … …  
    10198    $DET_ID   = $word:0
    10299    $ITER     = $word:1
    103     $TYPE    = $word:2
     100    $DET_TYPE = $word:2
    104101    $CLASS_ID = $word:3
    105102    $CAMERA   = $word:4
    106103
    107     stdout $CAMERA.$TYPE.$DET_ID.$ITER.$CLASS_ID.log
    108     stderr $CAMERA.$TYPE.$DET_ID.$ITER.$CLASS_ID.log
     104    # XXX add $WORKDIR/$LOG_DIR
     105    stdout $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log
     106    stderr $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log
    109107
    110108    # specify choice of remote host:
    … …  
    118116    options "$line"
    119117    # detrend_stack.pl --det_id 1 --iteration 0 --class_id isp --det_type bias --camera ISP || exit 1
    120     echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $TYPE --camera $CAMERA
    121     command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $TYPE --camera $CAMERA
     118    echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA
     119    command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA
    122120  end
    123121

  • trunk/ippTasks/pantasks.pro

    r9104 r9338  
    22$network = 1
    33$parallel = 0
    4 $scripts = /home/kiawe/eugene/src/panstarrs/ipp.full/ippTasks
     4$scripts = /home/kiawe/eugene/src/panstarrs/ipp/ippTasks
     5$WORKDIR = `ipp_workdir.pl`
    56
    67macro init
    … …  
    1112  exec rm -f *.stats
    1213  exec rm -f *.detproc.*.fits
     14  exec rm -f *.detproc.*.jpg
     15  exec rm -f *.detproc.*.list
    1316  exec rm -f *.detresid.*.fits
    1417  exec rm -f *.detresid.*.jpg
    1518  exec rm -f *.detresid.*.list
    16   exec rm -f isp.bias.*.fits
     19  exec rm -f ISP.bias.*.fits
    1720
    18   exec upload.sh
     21  exec rm -f ../bias/*.log
     22  exec rm -f ../bias/*.stats
     23  exec rm -f ../bias/*.detproc.*.fits
     24  exec rm -f ../bias/*.detproc.*.jpg
     25  exec rm -f ../bias/*.detproc.*.list
     26  exec rm -f ../bias/*.detresid.*.fits
     27  exec rm -f ../bias/*.detresid.*.jpg
     28  exec rm -f ../bias/*.detresid.*.list
     29  exec rm -f ../bias/ISP.bias.*.fits
     30
     31  exec rm -f ../dark/*.log
     32  exec rm -f ../dark/*.stats
     33  exec rm -f ../dark/*.detproc.*.fits
     34  exec rm -f ../dark/*.detproc.*.jpg
     35  exec rm -f ../dark/*.detproc.*.list
     36  exec rm -f ../dark/*.detresid.*.fits
     37  exec rm -f ../dark/*.detresid.*.jpg
     38  exec rm -f ../dark/*.detresid.*.list
     39  exec rm -f ../dark/ISP.dark.*.fits
     40
     41  exec rm -f ../flat/*.log
     42  exec rm -f ../flat/*.stats
     43  exec rm -f ../flat/*.detproc.*.fits
     44  exec rm -f ../flat/*.detproc.*.jpg
     45  exec rm -f ../flat/*.detproc.*.list
     46  exec rm -f ../flat/*.detresid.*.fits
     47  exec rm -f ../flat/*.detresid.*.jpg
     48  exec rm -f ../flat/*.detresid.*.list
     49  exec rm -f ../flat/ISP.flat.*.fits
    1950end
    2051
    … …  
    2354macro step1
    2455  init
     56  exec upload.sh
    2557  input $scripts/phase0.pro
    2658  run
    … …  
    2860
    2961# alala: dettool -tostack -simple
    30 macro step2.bias
     62macro define.bias
    3163  exec dettool -definebyquery -det_type bias -exp_type bias -camera isp
    32   input $scripts/detrend.process.pro
    3364  run
    3465end
    3566
    36 macro step2.dark
     67macro define.dark
    3768  exec dettool -definebyquery -det_type dark -exp_type dark -camera isp
    38   input $scripts/detrend.process.pro
    3969  run
    4070end
    4171
    42 macro step2.flat
     72macro define.flat
    4373  exec dettool -definebyquery -det_type flat -exp_type flat -camera isp
    44   input $scripts/detrend.process.pro
    4574  run
    4675end
    4776
    48 macro step2.both
     77macro define.all
    4978  exec dettool -definebyquery -det_type bias -exp_type bias -camera isp
    5079  exec dettool -definebyquery -det_type dark -exp_type dark -camera isp
    5180  exec dettool -definebyquery -det_type flat -exp_type flat -camera isp
    52   input $scripts/detrend.process.pro
    5381  run
    5482end
    5583
    56 macro step3
     84macro step2
     85  input $scripts/detrend.process.pro
    5786  input $scripts/detrend.stack.pro
    5887  input $scripts/detrend.norm.pro

  • trunk/ippTasks/phase0.pro

    r9305 r9338  
    9898    end
    9999
    100     # do something better with this output target
    101     stdout $EXP_TAG.$CLASS_ID.log
    102     stderr $EXP_TAG.$CLASS_ID.log
     100    $LOG_DIR = `dirname $URI`
     101    stdout $WORKDIR/$LOG_DIR/$EXP_TAG.$CLASS_ID.log
     102    stderr $WORKDIR/$LOG_DIR/$EXP_TAG.$CLASS_ID.log
    103103
    104104    # create the command line
    … …  
    204204    $EXP_TAG   = $word:0
    205205
    206     # do something better with this output target
     206    # XXX add $WORKDIR/$LOG_DIR
    207207    stdout $EXP_TAG.log
    208208    stderr $EXP_TAG.log
Note: See TracChangeset for help on using the changeset viewer.