IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jul 30, 2008, 10:42:33 AM (18 years ago)
Author:
Paul Price
Message:

First cut at task for magic stages using reworked process flow. No
doubt things will have to be fixed once the scripts are done, but this
should be a useful first cut.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/magic.pro

    r17653 r18783  
    88
    99### Initialise the books containing the tasks to do
     10book init magicToTree
    1011book init magicToProcess
    1112book init magicToMask
    12 book init magicToSkyfileMask
    1313
    1414### Database lists
    15 $magic_DB = 0
    16 
    17 ### Check status of warping tasks
    18 #macro warp.status
    19 #  book listbook warpInputExp
    20 #  book listbook warpPendingSkyCell
    21 #end
    22 
    23 ### Reset warping tasks
    24 #macro warp.reset
    25 #  book init warpInputExp
    26 #  book init warpPendingSkyCell
    27 #end
    28 
    29 ### Turn warping tasks on
    30 # macro warp.on
    31 #   task warp.exp.load
    32 #     active true
    33 #   end
    34 #   task warp.exp.run
    35 #     active true
    36 #   end
    37 #   task warp.skycell.load
    38 #     active true
    39 #   end
    40 #   task warp.skycell.run
    41 #     active true
    42 #   end
    43 # end
    44 #
    45 # ### Turn warping tasks off
    46 # macro warp.off
    47 #   task warp.exp.load
    48 #     active false
    49 #   end
    50 #   task warp.exp.run
    51 #     active false
    52 #   end
    53 #   task warp.skycell.load
    54 #     active false
    55 #   end
    56 #   task warp.skycell.run
    57 #     active false
    58 #   end
    59 # end
    60 
    61 # populate magicToProcess
    62 task           magic.toprocess
     15$magicTree_DB = 0
     16$magicProcess_DB = 0
     17$magicMask_DB = 0
     18
     19### Check status of tasks
     20macro warp.status
     21  book listbook magicToTree
     22  book listbook magicToProcess
     23  book listbook magicToMask
     24end
     25
     26### Reset tasks
     27macro warp.reset
     28  book init magicTree
     29  book init magicProcess
     30  book init magicMask
     31end
     32
     33### Turn tasks on
     34macro warp.on
     35  task magic.tree.load
     36    active true
     37  end
     38  task magic.tree.run
     39    active true
     40  end
     41  task magic.process.load
     42    active true
     43  end
     44  task magic.process.run
     45    active true
     46  end
     47  task magic.mask.load
     48    active true
     49  end
     50  task magic.mask.run
     51    active true
     52  end
     53end
     54
     55### Turn tasks off
     56macro warp.on
     57  task magic.tree.load
     58    active false
     59  end
     60  task magic.tree.run
     61    active false
     62  end
     63  task magic.process.load
     64    active false
     65  end
     66  task magic.process.run
     67    active false
     68  end
     69  task magic.mask.load
     70    active false
     71  end
     72  task magic.mask.run
     73    active false
     74  end
     75end
     76
     77task           magic.tree.load
    6378  host         local
    6479
    … …  
    6984
    7085  stdout NULL
    71   stderr $LOGSUBDIR/magic.toprocess.log
     86  stderr $LOGDIR/magic.tree.log
    7287
    7388  task.exec
    7489    if ($DB:n == 0)
    7590      option DEFAULT
    76       run = command magictool -toprocess -limit 20
    77       if ($VERBOSE > 1)
    78         echo command $run
    79       end
    80       command $run
     91      command magictool -totree -limit 20
    8192    else
    8293      # save the DB name for the exit tasks
    83       option $DB:$magic_DB
    84       command magictool -toprocess -limit 20 -dbname $DB:$magic_DB
    85       $magic_DB ++
    86       if ($magic_DB >= $DB:n) set magic_DB = 0
     94      option $DB:$magicTree_DB
     95      command magictool -totree -limit 20 -dbname $DB:$magicTree_DB
     96      $magicTree_DB ++
     97      if ($magicTree_DB >= $DB:n) set magicTree_DB = 0
    8798    end
    8899  end
    … …  
    91102  task.exit    0
    92103    # convert 'stdout' to book format
    93     ipptool2book stdout magicToProcess -key magic_id:node -uniq -setword dbname $options:0 -setword pantaskState INIT
     104    ipptool2book stdout magicTree -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT
    94105    if ($VERBOSE > 2)
    95       book listbook magicToProcess
     106      book listbook magicTree
    96107    end
    97108
    98109    # delete existing entries in the appropriate pantaskStates
    99     process_cleanup magicToProcess
     110    process_cleanup magicTree
    100111  end
    101112
    … …  
    111122end
    112123
    113 # task   magic.process
    114 #   periods      -poll $loadpoll
    115 #   periods      -exec $loadexec
    116 #   periods      -timeout 30
    117 #
    118 #   task.exec
    119 #   end
    120 #
    121 #   # default exit status
    122 #   task.exit    default
    123 #     process_exit magicToProcess $options:0 $JOB_STATUS
    124 #   end
    125 #
    126 #   # operation timed out?
    127 #   task.exit    timeout
    128 #     showcommand timeout
    129 #     book setword magicToProcess $options:0 pantaskState TIMEOUT
    130 #   end
    131 # end
    132 
    133 # populate magicToMask
    134 task           magic.tomask
     124task           magic.tree.run
     125  periods      -poll $RUNPOLL
     126  periods      -exec $RUNEXEC
     127  periods      -timeout 60
     128
     129  task.exec
     130    book npages magicTree -var N
     131    if ($N == 0) break
     132    if ($NETWORK == 0) break
     133   
     134    # look for new images (pantaskState == INIT)
     135    book getpage magicTree 0 -var pageName -key pantaskState INIT
     136    if ("$pageName" == "NULL") break
     137
     138    book setword magicTree $pageName pantaskState RUN
     139    book getword magicTree $pageName magic_id -var MAGIC_ID
     140    book getword magicTree $pageName exp_id -var EXP_ID
     141    book getword magicTree $pageName camera -var CAMERA
     142    book getword magicTree $pageName workdir -var WORKDIR_TEMPLATE
     143    book getword magicTree $pageName dbname -var DBNAME
     144    book getword magicTree $pageName tess_id -var TESS_DIR
     145    book getword magicTree $pageName ra -var RA
     146    book getword magicTree $pageName decl -var DEC
     147
     148#    set.host.for.camera $CAMERA $MAGIC_ID
     149#    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     150    host anyhost
     151    $WORKDIR = $WORKDIR_TEMPLATE
     152
     153    basename $TESS_DIR -var TESS_ID
     154    sprintf outroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
     155
     156    ## generate output log based on filerule (convert the URI to a PATH)
     157    $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot`
     158    if ("$logfile" == "")
     159      echo "WARNING: logfile not defined in magic.tree.run"
     160      break
     161    end
     162
     163    stdout $logfile
     164    stderr $logfile
     165    dirname $logfile -var outpath
     166    mkdir $outpath
     167
     168    $run = magic_tree.pl --magic_id $MAGIC_ID --camera $CAMERA --tess_id $TESS_DIR --ra $RA --dec $DEC --outroot $outroot --logfile $logfile
     169    add_standard_args run
     170
     171    # save the pageName for future reference below
     172    options $pageName
     173
     174    # create the command line
     175    if ($VERBOSE > 1)
     176      echo command $run
     177    end
     178    command $run
     179  end
     180
     181  # default exit status
     182  task.exit    default
     183    process_exit magicTree $options:0 $JOB_STATUS
     184  end
     185
     186  # operation timed out?
     187  task.exit    timeout
     188    showcommand timeout
     189    book setword magicTree $options:0 pantaskState TIMEOUT
     190  end
     191end
     192
     193task           magic.process.load
    135194  host         local
    136195
    … …  
    141200
    142201  stdout NULL
    143   stderr $LOGSUBDIR/magic.tomask.log
     202  stderr $LOGDIR/magic.process.log
    144203
    145204  task.exec
    146205    if ($DB:n == 0)
    147206      option DEFAULT
    148       run = command magictool -tomask -limit 20
    149       if ($VERBOSE > 1)
    150         echo command $run
    151       end
    152       command $run
     207      command magictool -toprocess -limit 20
    153208    else
    154209      # save the DB name for the exit tasks
    155       option $DB:$magic_DB
    156       command magictool -tomask -limit 20 -dbname $DB:$magic_DB
    157       $magic_DB ++
    158       if ($magic_DB >= $DB:n) set magic_DB = 0
     210      option $DB:$magicTree_DB
     211      command magictool -toprocess -limit 20 -dbname $DB:$magicProcess_DB
     212      $magicProcess_DB ++
     213      if ($magicProcess_DB >= $DB:n) set magicProcess_DB = 0
    159214    end
    160215  end
    … …  
    163218  task.exit    0
    164219    # convert 'stdout' to book format
    165     ipptool2book stdout magicToMask -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     220    ipptool2book stdout magicProcess -key magic_id:node -uniq -setword dbname $options:0 -setword pantaskState INIT
    166221    if ($VERBOSE > 2)
    167       book listbook magicToProcess
     222      book listbook magicProcess
    168223    end
    169224
    170225    # delete existing entries in the appropriate pantaskStates
    171     process_cleanup magicToMask
     226    process_cleanup magicProcess
    172227  end
    173228
    … …  
    183238end
    184239
    185 
    186 # populate magicToMask
    187 task           magic.toskyfilemask
     240task           magic.process.run
     241  periods      -poll $RUNPOLL
     242  periods      -exec $RUNEXEC
     243  periods      -timeout 60
     244
     245  task.exec
     246    book npages magicProcess -var N
     247    if ($N == 0) break
     248    if ($NETWORK == 0) break
     249   
     250    # look for new images (pantaskState == INIT)
     251    book getpage magicProcess 0 -var pageName -key pantaskState INIT
     252    if ("$pageName" == "NULL") break
     253
     254    book setword magicTree $pageName pantaskState RUN
     255    book getword magicTree $pageName magic_id -var MAGIC_ID
     256    book getword magicTree $pageName exp_id -var EXP_ID
     257    book getword magicTree $pageName node -var NODE
     258    book getword magicTree $pageName camera -var CAMERA
     259    book getword magicTree $pageName workdir -var WORKDIR_TEMPLATE
     260    book getword magicTree $pageName dbname -var DBNAME
     261
     262#    set.host.for.camera $CAMERA $MAGIC_ID
     263#    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     264    host anyhost
     265    $WORKDIR = $WORKDIR_TEMPLATE
     266
     267    basename $TESS_DIR -var TESS_ID
     268    sprintf outroot "%s/%s/%s.mgc.%s.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID $NODE
     269
     270    ## generate output log based on filerule (convert the URI to a PATH)
     271    $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot`
     272    if ("$logfile" == "")
     273      echo "WARNING: logfile not defined in magic.process.run"
     274      break
     275    end
     276
     277    stdout $logfile
     278    stderr $logfile
     279    dirname $logfile -var outpath
     280    mkdir $outpath
     281
     282    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --outroot $outroot --logfile $logfile
     283    add_standard_args run
     284
     285    # save the pageName for future reference below
     286    options $pageName
     287
     288    # create the command line
     289    if ($VERBOSE > 1)
     290      echo command $run
     291    end
     292    command $run
     293  end
     294
     295  # default exit status
     296  task.exit    default
     297    process_exit magicProcess $options:0 $JOB_STATUS
     298  end
     299
     300  # operation timed out?
     301  task.exit    timeout
     302    showcommand timeout
     303    book setword magicProcess $options:0 pantaskState TIMEOUT
     304  end
     305end
     306
     307task           magic.mask.load
    188308  host         local
    189309
    … …  
    194314
    195315  stdout NULL
    196   stderr $LOGSUBDIR/magic.toskyfilemask.log
     316  stderr $LOGDIR/magic.mask.log
    197317
    198318  task.exec
    199319    if ($DB:n == 0)
    200320      option DEFAULT
    201       run = command magictool -toskyfilemask -limit 20
    202       if ($VERBOSE > 1)
    203         echo command $run
    204       end
    205       command $run
     321      command magictool -tomask -limit 20
    206322    else
    207323      # save the DB name for the exit tasks
    208       option $DB:$magic_DB
    209       command magictool -toskyfilemask-limit 20 -dbname $DB:$magic_DB
    210       $magic_DB ++
    211       if ($magic_DB >= $DB:n) set magic_DB = 0
     324      option $DB:$magicMask_DB
     325      command magictool -tomask -limit 20 -dbname $DB:$magicMask_DB
     326      $magicMask_DB ++
     327      if ($magicMask_DB >= $DB:n) set magicMask_DB = 0
    212328    end
    213329  end
    … …  
    216332  task.exit    0
    217333    # convert 'stdout' to book format
    218     ipptool2book stdout magicToSkyfileMask -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     334    ipptool2book stdout magicMask -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT
    219335    if ($VERBOSE > 2)
    220       book listbook magicToProcess
     336      book listbook magicMask
    221337    end
    222338
    223339    # delete existing entries in the appropriate pantaskStates
    224     process_cleanup magicToSkyfileMask
     340    process_cleanup magicMask
    225341  end
    226342
    … …  
    235351  end
    236352end
     353
     354task           magic.mask.run
     355  periods      -poll $RUNPOLL
     356  periods      -exec $RUNEXEC
     357  periods      -timeout 60
     358
     359  task.exec
     360    book npages magicMask -var N
     361    if ($N == 0) break
     362    if ($NETWORK == 0) break
     363   
     364    # look for new images (pantaskState == INIT)
     365    book getpage magicMask 0 -var pageName -key pantaskState INIT
     366    if ("$pageName" == "NULL") break
     367
     368    book setword magicMask $pageName pantaskState RUN
     369    book getword magicMask $pageName magic_id -var MAGIC_ID
     370    book getword magicMask $pageName exp_id -var EXP_ID
     371    book getword magicMask $pageName camera -var CAMERA
     372    book getword magicMask $pageName workdir -var WORKDIR_TEMPLATE
     373    book getword magicMask $pageName dbname -var DBNAME
     374
     375#    set.host.for.camera $CAMERA $MAGIC_ID
     376#    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     377    host anyhost
     378    $WORKDIR = $WORKDIR_TEMPLATE
     379
     380    basename $TESS_DIR -var TESS_ID
     381    sprintf outroot "%s/%s/%s.mgc.%s.mask" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
     382
     383    ## generate output log based on filerule (convert the URI to a PATH)
     384    $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot`
     385    if ("$logfile" == "")
     386      echo "WARNING: logfile not defined in magic.tree.run"
     387      break
     388    end
     389
     390    stdout $logfile
     391    stderr $logfile
     392    dirname $logfile -var outpath
     393    mkdir $outpath
     394
     395    $run = magic_mask.pl --magic_id $MAGIC_ID --camera $CAMERA --outroot $outroot --logfile $logfile
     396    add_standard_args run
     397
     398    # save the pageName for future reference below
     399    options $pageName
     400
     401    # create the command line
     402    if ($VERBOSE > 1)
     403      echo command $run
     404    end
     405    command $run
     406  end
     407
     408  # default exit status
     409  task.exit    default
     410    process_exit magicMask $options:0 $JOB_STATUS
     411   end
     412
     413  # operation timed out?
     414  task.exit    timeout
     415    showcommand timeout
     416    book setword magicMask $options:0 pantaskState TIMEOUT
     417  end
     418end
     419
Note: See TracChangeset for help on using the changeset viewer.