IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Sep 16, 2009, 9:47:16 AM (17 years ago)
Author:
bills
Message:

Change magicdstool -todestreak to query the individual stages separately.
This greatly decreases the query time.
Add support for magicDSRun.state = 'goto_restore' and 'goto_censored' which
will restore destreaked files back to their original state.
Split up ippTasks/magic.pro into magic.pro and destreak.pro so that processing
can be easily magnaged separately

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/magic.pro

    r25021 r25419  
    1010book init magicToTree
    1111book init magicToProcess
    12 book init magicToDS
    13 book init magicDSToRevert
    1412
    1513### Database lists
    1614$magicToTree_DB = 0
    1715$magicToProcess_DB = 0
    18 $magicToDS_DB = 0
    19 $magicDSToRevert_DB = 0
    20 
    21 #list of stages
    22 $STAGE:n = 0
    23 list STAGE -add "raw"
    24 list STAGE -add "chip"
    25 list STAGE -add "camera"
    26 list STAGE -add "warp"
    27 list STAGE -add "diff"
    28 $magicDSRevertStage = 0
    2916
    3017### Check status of tasks
    … …  
    3219  book listbook magicToTree
    3320  book listbook magicToProcess
    34   book listbook magicToDS
    35   book listbook magicDSToRevert
    3621end
    3722
    … …  
    4025  book init magicToTree
    4126  book init magicToProcess
    42   book init magicToDS
    43   book init magicDSToRevert
    4427end
    4528
    … …  
    5841    active true
    5942  end
    60   task magic.destreak.load
    61     active true
    62   end
    63   task magic.destreak.run
    64     active true
    65   end
    66 end
    67 
    68 macro magic.ds.revert.on
    69     task magic.ds.revert.load
    70         active true
    71     end
    72     task magic.ds.revert.run
    73         active true
    74     end
    75 end
     43end
     44
    7645### Turn tasks off
    7746macro magic.off
    … …  
    8857    active false
    8958  end
    90   task magic.destreak.load
    91     active false
    92   end
    93   task magic.destreak.run
    94     active false
    95   end
    96 end
    97 
    98 macro magic.ds.revert.off
    99     task magic.ds.revert.load
    100         active false
    101     end
    102     task magic.ds.revert.run
    103         active false
    104     end
    105 end
    106 macro magic.ds.off
    107     task magic.destreak.load
    108         active false
    109     end
    110     task magic.destreak.run
    111         active false
    112     end
    113 end
    114 macro magic.ds.on
    115     task magic.destreak.load
    116         active true
    117     end
    118     task magic.destreak.run
    119         active true
    120     end
    121 end
    122 
    123 macro magic.ds.status
    124     echo magicToDS
    125     book listbook magicToDS
    126     echo ""
    127     echo magicDSToRevert
    128     book listbook magicDSToRevert
    129 end
    130 
    131 
     59end
    13260
    13361task           magic.tree.load
    … …  
    400328  end
    401329end
    402 
    403 task           magic.destreak.load
    404   host         local
    405 
    406   periods      -poll $LOADPOLL
    407   # this query can take a long time
    408   periods      -exec 10
    409   periods      -timeout 120
    410   npending     1
    411 
    412   stdout NULL
    413   stderr $LOGDIR/magic.destreak.log
    414 
    415   task.exec
    416     $run = magicdstool -todestreak -limit 120
    417     if ($DB:n == 0)
    418       option DEFAULT
    419     else
    420       # save the DB name for the exit tasks
    421       option $DB:$magicToDS_DB
    422       $run = $run -dbname $DB:$magicToDS_DB
    423       $magicToDS_DB ++
    424       if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0
    425     end
    426     add_poll_args run
    427     add_poll_labels run
    428     command $run
    429   end
    430 
    431   # success
    432   task.exit    0
    433     # convert 'stdout' to book format
    434     ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT
    435     if ($VERBOSE > 2)
    436       book listbook magicToDS
    437     end
    438 
    439     # delete existing entries in the appropriate pantaskStates
    440     process_cleanup magicToDS
    441   end
    442 
    443   # locked list
    444   task.exit    default
    445     showcommand failure
    446   end
    447 
    448   task.exit    crash
    449     showcommand crash
    450   end
    451 
    452   # operation times out?
    453   task.exit    timeout
    454     showcommand timeout
    455   end
    456 end
    457 
    458 task           magic.destreak.run
    459   periods      -poll $RUNPOLL
    460   periods      -exec $RUNEXEC
    461   periods      -timeout 60
    462 
    463   task.exec
    464     book npages magicToDS -var N
    465     if ($N == 0) break
    466     if ($NETWORK == 0) break
    467    
    468     # look for new images (pantaskState == INIT)
    469     book getpage magicToDS 0 -var pageName -key pantaskState INIT
    470     if ("$pageName" == "NULL") break
    471 
    472     book setword magicToDS $pageName pantaskState RUN
    473     book getword magicToDS $pageName exp_id -var EXP_ID
    474     book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID
    475     book getword magicToDS $pageName camera -var CAMERA
    476     book getword magicToDS $pageName streaks_uri -var STREAKS
    477     book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS
    478     book getword magicToDS $pageName stage -var STAGE
    479     book getword magicToDS $pageName stage_id -var STAGE_ID
    480     book getword magicToDS $pageName component -var COMPONENT
    481     book getword magicToDS $pageName uri -var URI
    482     book getword magicToDS $pageName path_base -var PATH_BASE
    483     book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE
    484     book getword magicToDS $pageName outroot -var OUTROOT
    485     book getword magicToDS $pageName recoveryroot -var RECROOT
    486     book getword magicToDS $pageName re_place -var REPLACE
    487     book getword magicToDS $pageName dbname -var DBNAME
    488 
    489     sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
    490 
    491     substr $COMPONENT 0 3 COMP_HEAD
    492     if ("$COMP_HEAD" == "sky")
    493         set.host.for.skycell $COMPONENT
    494     else
    495         # assume component is a class_id, if not we will default to anyhost
    496         set.host.for.camera $CAMERA $COMPONENT
    497     end
    498 
    499     # TODO: do not add recoveryroot or replace if they are null or zero
    500 
    501     $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE
    502 
    503     add_standard_args run
    504 
    505     # save the pageName for future reference below
    506     options $pageName
    507 
    508     # create the command line
    509     if ($VERBOSE > 1)
    510       echo command $run
    511     end
    512     command $run
    513   end
    514 
    515   # default exit status
    516   task.exit    0
    517     process_exit magicToDS $options:0 $JOB_STATUS
    518    end
    519 
    520   # locked list
    521   task.exit    default
    522     showcommand failure
    523     process_exit magicToDS $options:0 $JOB_STATUS
    524   end
    525 
    526   task.exit    crash
    527     showcommand crash
    528     book setword magicToDS $options:0 pantaskState CRASH
    529   end
    530 
    531   # operation timed out?
    532   task.exit    timeout
    533     showcommand timeout
    534     book setword magicToDS $options:0 pantaskState TIMEOUT
    535   end
    536 end
    537 
    538 task           magic.ds.revert.load
    539   host         local
    540 
    541   periods      -poll $LOADPOLL
    542   periods      -exec $LOADEXEC
    543   periods      -timeout 20
    544   npending     1
    545   active       false
    546 
    547   stdout NULL
    548   stderr $LOGDIR/magic.ds.revert.log
    549 
    550   task.exec
    551     $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage
    552     $magicDSRevertStage ++
    553     if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0
    554 
    555     if ($DB:n == 0)
    556       option DEFAULT
    557     else
    558 
    559       # save the DB name for the exit tasks
    560       option $DB:$magicDSToRevert_DB
    561       $run = $run -dbname $DB:$magicDSToRevert_DB
    562 
    563       # only bump database number after we have gone through all of the stages
    564       if ($magicDSRevertStage == 0)
    565              $magicDSToRevert_DB ++
    566       end
    567       if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0
    568     end
    569     add_poll_args run
    570     add_poll_labels run
    571     command $run
    572   end
    573 
    574   # success
    575   task.exit    0
    576     # convert 'stdout' to book format
    577     ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT
    578     if ($VERBOSE > 2)
    579       book listbook magicDSToRevert
    580     end
    581 
    582     # delete existing entries in the appropriate pantaskStates
    583     process_cleanup magicDSToRevert
    584   end
    585 
    586   # locked list
    587   task.exit    default
    588     showcommand failure
    589   end
    590 
    591   task.exit    crash
    592     showcommand crash
    593   end
    594 
    595   # operation times out?
    596   task.exit    timeout
    597     showcommand timeout
    598   end
    599 end
    600 
    601 task           magic.ds.revert.run
    602   periods      -poll $RUNPOLL
    603   periods      -exec $RUNEXEC
    604   periods      -timeout 60
    605   active       false
    606 
    607   task.exec
    608     book npages magicDSToRevert -var N
    609     if ($N == 0) break
    610     if ($NETWORK == 0) break
    611    
    612     # look for new images (pantaskState == INIT)
    613     book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT
    614     if ("$pageName" == "NULL") break
    615 
    616     book setword magicDSToRevert $pageName pantaskState RUN
    617     book getword magicDSToRevert $pageName exp_id -var EXP_ID
    618     book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID
    619     book getword magicDSToRevert $pageName camera -var CAMERA
    620     book getword magicDSToRevert $pageName stage -var STAGE
    621     book getword magicDSToRevert $pageName stage_id -var STAGE_ID
    622     book getword magicDSToRevert $pageName component -var COMPONENT
    623     book getword magicDSToRevert $pageName path_base -var PATH_BASE
    624     book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE
    625     book getword magicDSToRevert $pageName outroot -var OUTROOT
    626     book getword magciDSToRevert $pageName bytes -var BYTES
    627     book getword magciDSToRevert $pageName md5sum -var md5sum
    628 #    book getword magicDSToRevert $pageName recoveryroot -var RECROOT
    629     book getword magicDSToRevert $pageName re_place -var REPLACE
    630     book getword magicDSToRevert $pageName dbname -var DBNAME
    631 
    632     sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
    633 
    634     substr $COMPONENT 0 3 COMP_HEAD
    635     if ("$COMP_HEAD" == "sky")
    636         set.host.for.skycell $COMPONENT
    637     else
    638         # assume component is a class_id, if not we will default to anyhost
    639         set.host.for.camera $CAMERA $COMPONENT
    640     end
    641 
    642     $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE
    643 
    644     add_standard_args run
    645 
    646     # save the pageName for future reference below
    647     options $pageName
    648 
    649     # create the command line
    650     if ($VERBOSE > 1)
    651       echo command $run
    652     end
    653     command $run
    654   end
    655 
    656   # default exit status
    657   task.exit    0
    658     process_exit magicDSToRevert $options:0 $JOB_STATUS
    659    end
    660 
    661   # locked list
    662   task.exit    default
    663     showcommand failure
    664     process_exit magicDSToRevert $options:0 $JOB_STATUS
    665   end
    666 
    667   task.exit    crash
    668     showcommand crash
    669     book setword magicDSToRevert $options:0 pantaskState CRASH
    670   end
    671 
    672   # operation timed out?
    673   task.exit    timeout
    674     showcommand timeout
    675     book setword magicDSToRevert $options:0 pantaskState TIMEOUT
    676   end
    677 end
    678 
Note: See TracChangeset for help on using the changeset viewer.