IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 8691


Ignore:
Timestamp:
Aug 29, 2006, 2:28:33 PM (20 years ago)
Author:
eugene
Message:

matching detrend and phase0 PanTasks scripts with current pXtools commands

Location:
trunk/ippTasks
Files:
2 added
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/detrend.assess.pro

    r8193 r8691  
    11
    2 ## this file contains the tasks for running the detrend normalization stage
     2## this file contains the tasks for running the detrend processing stage
    33
    4 ## these tasks use the queue DetrendRunToAssess
     4# dettool -toresidexp -simple
     5# 1      0    bias 850131b T
     6# DET_ID ITER TYPE EXP_KEY INCLUDE         
     7
     8# detrend_reject_imfile.pl 1      0    850131b bias
     9# detrend_reject_imfile.pl DET_ID ITER EXP_KEY TYPE
     10
     11## these tasks use the queue DetrendResidExp
     12## the DetrendResidExp queue contains:
     13## STATE DET_ID ITER TYPE EXP_KEY INCLUDE
    514
    615# select images ready for copy
    7 # new entries are added to queue DetrendRunToAssess
     16# new entries are added to queue DetrendResidExp
    817# compare the new list with the ones already selected
    918task           dettool.assess.load
    10   command      dettool -stacked -runs
     19  command      dettool -toresidexp -simple
    1120  host         local
    1221
    … …  
    1726  # success
    1827  task.exit    0
    19     # XXX is it necessary for these to be local?
    20     # drop after this task macro is done?
    2128    local i Nqueue
    2229
    … …  
    2633    for i 0 $Nqueue
    2734      queuepop stdout -var line
    28       queuepush DetrendRunToAssess -uniq -key 1:2:4 "NEW $line"
     35      list word -split $line
     36      $DET_ID   = $word:0
     37      $ITER     = $word:1
     38      $TYPE     = $word:2
     39      $EXP_KEY  = $word:3
     40      $INCLUDE  = $word:4
     41      queuepush DetrendResidExp -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $EXP_KEY $INCLUDE"
    2942    end
    3043
    3144    # delete existing entries which are DONE
    32     queuedrop DetrendRunToAssess -key 0 DONE
     45    queuedrop DetrendResidExp -key 0 DONE
    3346  end
    3447
    … …  
    5063
    5164# copy new images, sending job to desired host
    52 task           dettool.assess.process
     65task           dettool.resid.process
    5366  periods      -poll 0.5
    5467  periods      -exec 1.0
    … …  
    5669
    5770  task.exec
    58     queuesize DetrendRunToAssess -var N
    59    
     71    queuesize DetrendResidExp -var N
    6072    if ($N == 0) break
    6173    if ($network == 0) break
    6274   
    63     # look for new images on the internal DetrendRunToAssess queue
     75    # look for new images on the internal DetrendResidExp queue
    6476    # caution with these 'if' statements: syntax errors
    6577    # will make the task fail without given a good status
    66     queuepop DetrendRunToAssess -var line -key 0 NEW
     78    queuepop DetrendResidExp -var line -key 0 NEW
    6779    if ("$line" == "NULL") break
    6880
    6981    strpop line state
    70     queuepush DetrendRunToAssess -replace -key 0 "RUN $line"
     82    queuepush DetrendResidExp -replace -key 1:2:4 "RUN $line"
     83
     84    ## the DetrendResidExp queue contains:
     85    ## STATE DET_ID ITER TYPE EXP_KEY INCLUDE
     86    ## NOTE: the queue has the STATE prepended
     87    list word -split $line
     88    $DET_ID   = $word:1
     89    $ITER     = $word:2
     90    $TYPE     = $word:3
     91    $EXP_KEY  = $word:4
     92    $INCLUDE  = $word:5
    7193
    7294    # specify choice of remote host:
    73     # 'anyhost' specified random remote host
    74     # if we needed to specify a host explicitly, use a command like
    75     # one of the following.  the -required forces execution on the
    76     # named host, and blocks the task until the host is available
    77     # host kiawe
    78     # host alala -required
    7995    host anyhost
    80 
    81     # choose the default recipe appropriate to this type of file
    82     $recipe = `get.recipe $type`
    8396
    8497    # create example job options as a demonstration
    8598    options "$line"
    86 
    87     # we can put the dettool and ppMerge commands here, or wrap them in a single script
    88     # $urilist = `dettool -stacked -det_id $det_id -iteration $iteration -chip"
    89     # $uri_out = ???
    90     # command ppNorm $uri_out $uri $uri ... -recipe PPMERGE $recipe ???
    91     command ASSESS?? $uri_out -det_id $det_id -iteration $iteration -recipe PPMERGE $recipe
     99    # detrend_reject_imfile.pl 1      0    850131b bias
     100    # detrend_reject_imfile.pl DET_ID ITER EXP_KEY TYPE
     101    command detrend_reject_imfile.pl $DET_ID $ITER $EXP_KEY $TYPE
    92102  end
    93103
    94104  # success
    95105  task.exit 0
    96     # XXX need to parse the options to identify the values below (if not part of cmdline?)
    97     exec dettool -addstacked -det_id $det_id -iteration $iteration -class_id $class_id -procrecipe $recipe
    98     queuepush DetrendRunToAssess -replace -key 1:2:4 "DONE $options:0"
     106    queuepush DetrendResidExp -replace -key 1:2:4 "DONE $options:0"
    99107  end
    100108
    101109  # default exit status
    102110  task.exit    default
    103     queuepush DetrendRunToAssess -replace -key 0 "FAIL $options:0"
     111    queuepush DetrendResidExp -replace -key 1:2:4 "FAIL $options:0"
    104112  end
    105113
    106114  # operation times out?
    107115  task.exit    timeout
    108     queuepush DetrendRunToAssess -replace -key 0 "FAIL $options:0"
     116    queuepush DetrendResidExp -replace -key 1:2:4 "FAIL $options:0"
    109117  end
    110118end

  • trunk/ippTasks/detrend.process.pro

    r8193 r8691  
     1## this file contains the tasks for running the detrend processing stage
    12
    2 ## this file contains the tasks for running the detrend processing stage
     3# query for pending imfiles to process:
     4# dettool -raw -simple
     5# 1 bias 850131b fpa mc 850131b.fits BIAS r.MP9601 1.656000 5.496118 0.349071 0.000000 1155.934692 107.037283 46.214947 37.160000 105.480000 -133.067001 0.000000
     6# this returns many columns for manual operations. we strip all but the first few:
     7
     8# process a single imfile
     9# detrend_process.pl 1 850131b mc bias 850131b.fits 850131b
    310
    411## these tasks use the queue DetrendImfilesToProcess
    512## the DetrendImfilesToProcess queue contains:
    6 ## STATE .....
     13## STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI
    714
    815# select images ready for copy
    … …  
    1017# compare the new list with the ones already selected
    1118task           dettool.raw.load
    12   command      dettool -raw
     19  command      dettool -raw -simple
    1320  host         local
    1421
    … …  
    2936    for i 0 $Nqueue
    3037      queuepop stdout -var line
    31       queuepush DetrendImfilesToProcess -uniq -key 1:2:4 "NEW $line"
     38      list word -split $line
     39      $DET_ID   = $word:0
     40      $TYPE     = $word:1
     41      $EXP_KEY  = $word:2
     42      $CLASS    = $word:3
     43      $CLASS_ID = $word:4
     44      $URI      = $word:5
     45      queuepush DetrendImfilesToProcess -uniq -key 1:3:5 "NEW $DET_ID $TYPE $EXP_KEY $CLASS $CLASS_ID $URI"
    3246    end
    3347
    … …  
    7084
    7185    strpop line state
    72     queuepush DetrendImfilesToProcess -replace -key 0 "RUN $line"
     86    queuepush DetrendImfilesToProcess -replace -key 1:3:5 "RUN $line"
     87
     88    ## the DetrendImfilesToProcess queue contains:
     89    # STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI
     90    ## NOTE: the queue has the STATE prepended
     91    list word -split $line
     92    $DET_ID   = $word:1
     93    $TYPE     = $word:2
     94    $EXP_KEY  = $word:3
     95    $CLASS    = $word:4
     96    $CLASS_ID = $word:5
     97    $URI      = $word:6
    7398
    7499    # specify choice of remote host:
    75     # 'anyhost' specified random remote host
    76     # if we needed to specify a host explicitly, use a command like
    77     # one of the following.  the -required forces execution on the
    78     # named host, and blocks the task until the host is available
    79     # host kiawe
    80     # host alala -required
    81100    host anyhost
    82 
    83     # choose the default recipe appropriate to this type of file
    84     $recipe = `get.recipe $type`
    85101
    86102    # create example job options as a demonstration
    87103    options "$line"
    88     command ppImage -file $uri $uri_out -recipe PPIMAGE $recipe
     104    # ~/ipp/helpers/detrend_process.pl 1 850131b mc bias 850131b.fits 850131b
     105    command detrend_process.pl $DET_ID $EXP_KEY $CLASS_ID $TYPE $URI $EXP_KEY
    89106  end
    90107
    91108  # success
    92109  task.exit 0
    93     # XXX need to parse the options to identify the values below (if not part of cmdline?)
    94     exec dettool -addprocessed -det_id $det_id -exp_id $exp_id -class_id $class_id -uri $output_uri -procrecipe $recipe
    95     queuepush DetrendImfilesToProcess -replace -key 1:2:4 "DONE $options:0"
     110    queuepush DetrendImfilesToProcess -replace -key 1:3:5 "DONE $options:0"
    96111  end
    97112
    98113  # default exit status
    99114  task.exit    default
    100     queuepush DetrendImfilesToProcess -replace -key 0 "FAIL $options:0"
     115    queuepush DetrendImfilesToProcess -replace -key 1:3:5 "FAIL $options:0"
    101116  end
    102117
    103118  # operation times out?
    104119  task.exit    timeout
    105     queuepush DetrendImfilesToProcess -replace -key 0 "FAIL $options:0"
     120    queuepush DetrendImfilesToProcess -replace -key 1:3:5 "FAIL $options:0"
    106121  end
    107122end

  • trunk/ippTasks/detrend.resid.pro

    r8193 r8691  
    22## this file contains the tasks for running the detrend processing stage
    33
     4# dettool -toresid -simple
     5# 0    bias 1      850131b mc       o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582
     6# ITER TYPE DET_ID EXP_KEY CLASS_ID URI-IN           
     7
     8# detrend_create_resid.pl 1      0    850131b mc       bias bias_mc_1_0.fit o_850131b.mc.fit 850131b
     9# detrend_create_resid.pl DET_ID ITER EXP_KEY CLASS_ID TYPE (URI-DET)       (URL-IN)         (EXP_KEY)
     10# URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit
     11
    412## these tasks use the queue DetrendResidImfiles
    513## the DetrendResidImfiles queue contains:
    6 ## STATE .....
     14## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN
    715
    816# select images ready for copy
    … …  
    1018# compare the new list with the ones already selected
    1119task           dettool.resid.load
    12   command      dettool -resid
     20  command      dettool -toresid -simple
    1321  host         local
    1422
    … …  
    1624  periods      -exec 5
    1725  periods      -timeout 30
    18   trange       18:00 23:59
    1926
    2027  # success
    … …  
    2936    for i 0 $Nqueue
    3037      queuepop stdout -var line
    31       queuepush DetrendResidImfiles -uniq -key 1:2:4 "NEW $line"
     38      list word -split $line
     39      $ITER     = $word:0
     40      $TYPE     = $word:1
     41      $DET_ID   = $word:2
     42      $EXP_KEY  = $word:3
     43      $CLASS_ID = $word:4
     44      $URI_IN   = $word:5
     45      queuepush DetrendResidImfiles -uniq -key 1:3:4:5 "NEW $ITER $TYPE $DET_ID $EXP_KEY $CLASS_ID $URI_IN"
    3246    end
    3347
    … …  
    7084
    7185    strpop line state
    72     queuepush DetrendResidImfiles -replace -key 0 "RUN $line"
     86    queuepush DetrendResidImfiles -replace -key 1:3:4:5 "RUN $line"
     87
     88    ## the DetrendResidImfiles queue contains:
     89    ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN
     90    ## NOTE: the queue has the STATE prepended
     91    list word -split $line
     92    $ITER     = $word:1
     93    $TYPE     = $word:2
     94    $DET_ID   = $word:3
     95    $EXP_KEY  = $word:4
     96    $CLASS_ID = $word:5
     97    $URI_IN   = $word:6
    7398
    7499    # specify choice of remote host:
    75     # 'anyhost' specified random remote host
    76     # if we needed to specify a host explicitly, use a command like
    77     # one of the following.  the -required forces execution on the
    78     # named host, and blocks the task until the host is available
    79     # host kiawe
    80     # host alala -required
    81100    host anyhost
    82 
    83     # choose the default recipe appropriate to this type of file
    84     $recipe = `get.recipe $type`
    85101
    86102    # create example job options as a demonstration
    87103    options "$line"
    88 
    89     # need to choose different variations based on the type
    90     # need to supply the matched detrend images...
    91     command ppImage -file $uri $uri_out -recipe PPIMAGE $recipe -bias $master
    92     command ppImage -file $uri $uri_out -recipe PPIMAGE $recipe -flat $master
     104    # detrend_create_resid.pl DET_ID ITER EXP_KEY CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_KEY ???)
     105    $URL_DET = $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit
     106    command detrend_create_resid.pl $DET_ID $ITER $EXP_KEY $CLASS_ID $TYPE $URI_DET $URL_IN $EXP_KEY
    93107  end
    94108
    95109  # success
    96110  task.exit 0
    97     # XXX need to parse the options to identify the values below (if not part of cmdline?)
    98     exec dettool -addprocessed -det_id $det_id -exp_id $exp_id -class_id $class_id -uri $output_uri -procrecipe $recipe
    99     queuepush DetrendResidImfiles -replace -key 1:2:4 "DONE $options:0"
     111    queuepush DetrendResidImfiles -replace -key 1:3:4:5 "DONE $options:0"
    100112  end
    101113
    102114  # default exit status
    103115  task.exit    default
    104     queuepush DetrendResidImfiles -replace -key 0 "FAIL $options:0"
     116    queuepush DetrendResidImfiles -replace -key 1:3:4:5 "FAIL $options:0"
    105117  end
    106118
    107119  # operation times out?
    108120  task.exit    timeout
    109     queuepush DetrendResidImfiles -replace -key 0 "FAIL $options:0"
     121    queuepush DetrendResidImfiles -replace -key 1:3:4:5 "FAIL $options:0"
    110122  end
    111123end

  • trunk/ippTasks/detrend.stack.pro

    r8193 r8691  
    1 
    21## this file contains the tasks for running the detrend stacking stage
    32
     3# get a list of imfiles to stack
     4# dettool -tostack -simple
     5# 1 0 bias mc
     6# DET_ID ITER TYPE CLASS_ID
     7
     8# function to stack the imfiles:
     9# detrend_stack.pl 1 0 mc bias
     10
    411## these tasks use the queue DetrendClassIDtoStack
     12
     13## these tasks use the queue DetrendImfilesToProcess
     14## the DetrendClassIDtoStack queue contains:
     15## STATE DET_ID ITER TYPE CLASS_ID
    516
    617# select images ready for copy
    … …  
    819# compare the new list with the ones already selected
    920task           dettool.stack.load
    10   command      dettool -chip -processed
     21  command      dettool -tostack -simple
    1122  host         local
    1223
    … …  
    2233
    2334    # compare output with newImage queue
    24     # only add entries which don't exist in queue
     35    # only add entries which do not exist in queue
    2536    queuesize stdout -var Nqueue
    2637    for i 0 $Nqueue
    2738      queuepop stdout -var line
    28       queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $line"
     39      list word -split $line
     40      $DET_ID   = $word:0
     41      $ITER     = $word:1
     42      $TYPE     = $word:2
     43      $CLASS_ID = $word:3
     44      queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID"
    2945    end
    3046
    … …  
    6884
    6985    strpop line state
    70     queuepush DetrendClassIDtoStack -replace -key 0 "RUN $line"
     86    queuepush DetrendClassIDtoStack -replace -key 1:2:4 "RUN $line"
     87
     88    # the DetrendClassIDtoStack queue contains:
     89    # STATE DET_ID ITER TYPE CLASS_ID
     90    ## NOTE: the queue has the STATE prepended
     91    list word -split $line
     92    $DET_ID   = $word:1
     93    $ITER     = $word:2
     94    $TYPE     = $word:3
     95    $CLASS_ID = $word:4
    7196
    7297    # specify choice of remote host:
    73     # 'anyhost' specified random remote host
    74     # if we needed to specify a host explicitly, use a command like
    75     # one of the following.  the -required forces execution on the
    76     # named host, and blocks the task until the host is available
    77     # host kiawe
    78     # host alala -required
    7998    host anyhost
    80 
    81     # choose the default recipe appropriate to this type of file
    82     $recipe = `get.recipe $type`
    8399
    84100    # create example job options as a demonstration
    85101    options "$line"
    86 
    87     # we can put the dettool and ppMerge commands here, or wrap them in a single script
    88     # $urilist = `dettool -processed -det_id $det_id -iteration $iteration -chip"
    89     # $uri_out = ???
    90     # command ppMerge $uri_out $uri $uri ... -recipe PPMERGE $recipe
    91     command ppMerge.dettool $uri_out -det_id $det_id -iteration $iteration -recipe PPMERGE $recipe
     102    # detrend_stack.pl 1 0 mc bias
     103    command detrend_stack.pl $DET_ID $ITER $CLASS_ID $TYPE
    92104  end
    93105
    94106  # success
    95107  task.exit 0
    96     # XXX need to parse the options to identify the values below (if not part of cmdline?)
    97     exec dettool -addstacked -det_id $det_id -iteration $iteration -class_id $class_id -procrecipe $recipe
    98108    queuepush DetrendClassIDtoStack -replace -key 1:2:4 "DONE $options:0"
    99109  end
    … …  
    101111  # default exit status
    102112  task.exit    default
    103     queuepush DetrendClassIDtoStack -replace -key 0 "FAIL $options:0"
     113    queuepush DetrendClassIDtoStack -replace -key 1:2:4 "FAIL $options:0"
    104114  end
    105115
    106116  # operation times out?
    107117  task.exit    timeout
    108     queuepush DetrendClassIDtoStack -replace -key 0 "FAIL $options:0"
     118    queuepush DetrendClassIDtoStack -replace -key 1:2:4 "FAIL $options:0"
    109119  end
    110120end
Note: See TracChangeset for help on using the changeset viewer.