IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 14607


Ignore:
Timestamp:
Aug 22, 2007, 2:10:20 PM (19 years ago)
Author:
jhoblitt
Message:

incomplete attempt to covert to using books instead of queues

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/summit.copy.pro

    r14601 r14607  
    11## PanTasks scripts for Summit Copy
    22
    3 $uri        = http://otis/ds/skyprobe/
    4 $camera     = skyprobe
    5 $telescope  = ps1
     3#$uri        = http://otis/ds/skyprobe/
     4$uri        = http://otis4.ifa.hawaii.edu/ds/allskycam/
     5$camera     = allskycam
     6$telescope  = zl
    67$workdir    = /home/moanui/jhoblitt/workdir
    78
    … …  
    1011end
    1112
    12 # queue for summit exps that need to be queried
    13 queueinit pzPendingExp
    14 # queue for summit imfiles that need to be downloaded
    15 queueinit pzPendingImfile
    16 # queue for downloaded files that need to registered with pztool -copydone
    17 queueinit pzCopyDone
     13if ($?network == 0)
     14  $network = 1
     15end
     16
     17# book for summit exps that need to be queried
     18book init pzPendingExp
     19# book for summit imfiles that need to be downloaded
     20book init pzPendingImfile
    1821
    1922# run pzgetexp periodically to populate pzPendingExp
    … …  
    3134
    3235  task.exit     default
    33     echo "phaseZ pzgetexp: failure"
     36    showcommand failure
    3437  end
    3538  task.exit     timeout
    36     echo "phaseZ pzgetexp: timeout"
     39    showcommand timeout
    3740  end
    3841end
    3942
    40 # build a queue of exps/filesetids that need to be queried
     43# build a book of exps/filesetids that need to be queried
    4144task pztool.pendingexp
    4245    command      pztool -pendingexp -simple
    … …  
    4952
    5053    # success
    51     task.exit    0
    52         local i Nstdout
    53         queuesize stdout -var Nstdout
    54         for i 0 $Nstdout
    55             queuepop stdout -var line
    56 #            list word -split $line
    57 #            $exp_id     = $word:0
    58 #            $camera     = $word:1
    59 #            $telescope  = $word:2
    60 #            $dateobs    = $word:3
    61 #            $exp_type   = $word:4
    62 #            $uri        = $word:5
    63 #            $imfiles    = $word:6
    64 #            queuepush pzPendingExp -uniq -key 1:2:3 "NEW $uri $exp_id $camera $telescope $dateobs $exp_type $uri $imfiles"
    65             queuepush pzPendingExp -uniq -key 1:2:3 "NEW $line"
    66         end
    67 
    68         queuedrop pzPendingExp -key 0 DONE
    69     end
    70 
    71     task.exit     default
    72         echo "phaseZ pending exp: failure"
    73     end
    74     task.exit     timeout
    75         echo "phaseZ pending exp: timeout"
     54    task.exit 0
     55        # convert 'stdout' to book format
     56#        ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT
     57        ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword pantaskState INIT
     58
     59        # delete existing entries in the appropriate pantaskStates
     60        process_cleanup pzPendingExp
     61    end
     62
     63    task.exit default
     64        showcommand failure
     65    end
     66    task.exit timeout
     67        showcommand timeout
    7668    end
    7769end
    … …  
    8476
    8577    task.exec
    86         queuesize pzPendingExp -var N
     78        if ($network == 0) break
     79
     80        book npages pzPendingExp -var N
    8781        if ($N == 0)
    8882            periods      -exec     20
    8983            break
    9084        end
    91 
    9285        periods      -exec     0.2
    9386
    94         # get a line out of the queue
    95         queuepop pzPendingExp -var line -key 0 NEW
    96 
    97         # take the state out of the line
    98         strpop line state
    99 
    100         # put the line back into the queue with a stea of RUN
    101         queuepush pzPendingExp -replace -key 1:2:3 "RUN $line"
    102 
    103         # the STATE has already been removed from the line
    104         list word -split $line
    105         $exp_id     = $word:0
    106         $camera     = $word:1
    107         $telescope  = $word:2
    108         $dateobs    = $word:3
    109         $exp_type   = $word:4
    110         $uri        = $word:5
    111         $imfiles    = $word:6
    112 
    113         # specify choice of remote host:(need to choose based on chips)
    114 #        if ($parallel)
    115 #            host anyhost
    116 #        else
    117 #            host local
    118 #        end
    119        
    120         # store the current line
    121         options "$line"
    122 
    123         echo command pzgetimfiles -uri $uri -filesetid $exp_id -inst $camera -telescope $telescope
    124         command pzgetimfiles -uri $uri -filesetid $exp_id -inst $camera -telescope $telescope
     87        # find an exp that needs imfiles fetched
     88        book getpage pzPendingExp 0 -var pageName -key pantaskState INIT
     89        if ($pageName == NULL) break
     90
     91
     92        # set that exp to run
     93        book setword pzPendingExp $pageName pantaskState RUN
     94
     95        book getword pzPendingExp $pageName exp_id -var EXP_ID
     96        book getword pzPendingExp $pageName camera -var CAMERA
     97        book getword pzPendingExp $pageName telescope -var TELESCOPE
     98        book getword pzPendingExp $pageName dateobs -var DATEOBS
     99        book getword pzPendingExp $pageName exp_type -var EXP_TYPE
     100        book getword pzPendingExp $pageName uri -var URI
     101        book getword pzPendingExp $pageName imfiles -var IMFILES
     102
     103        # store the current page
     104        options $pageName
     105
     106        $run = pzgetimfiles -uri $URI -filesetid $EXP_ID -inst $CAMERA -telescope $TELESCOPE
     107
     108        # create the command line
     109        if ($VERBOSE > 1)
     110          echo command $run
     111        end
     112        command $run
    125113    end
    126114
    127115    # success
    128116    task.exit 0
    129         queuepush pzPendingExp -replace -key 1:2:3 "DONE $options:0"
     117        process_exit pzPendingExp $options:0 $JOB_STATUS
    130118    end
    131119
    132120    task.exit default
    133         queuepush pzPendingExp -replace -key 1:2:3 "FAIL $options:0"
     121        showcommand failure
     122        book setword pzPendingExp $options:0 pantaskState $JOB_STATUS
    134123    end
    135124
    136125    task.exit timeout
    137         queuepush pzPendingExp -replace -key 1:2:3 "TIMEOUT $options:0"
    138     end
    139 end
    140 
    141 # build a queue of imfiles/files that need to be downloaded
     126        showcommand timeout
     127        book setword pzPendingExp $options:0 pantaskState TIMEOUT
     128    end
     129end
     130
     131
     132# build a book of imfiles/files that need to be downloaded
    142133task pztool.pendingimfile
    143134    command      pztool -pendingimfile -simple
    … …  
    151142    # success
    152143    task.exit    0
    153         local i Nstdout
    154         queuesize stdout -var Nstdout
    155         for i 0 $Nstdout
    156             queuepop stdout -var line
    157 #            list word -split $line
    158 #            $exp_id     = $word:0
    159 #            $camera     = $word:1
    160 #            $telescope  = $word:2
    161 #            $file_id    = $word:3
    162 #            $bytes      = $word:4
    163 #            $md5sum     = $word:5
    164 #            $class      = $word:6
    165 #            $class_id   = $word:7
    166 #            $uri        = $word:8
    167 #            $exp_tag    = $word:9
    168 
    169             queuepush pzPendingImfile -uniq -key 1:2:3:7:8 "NEW $line"
    170         end
    171 
    172         # delete existing entries which are DONE
    173         queuedrop pzPendingImfile -key 0 DONE
     144        # convert 'stdout' to book format
     145#        ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT
     146        ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope -uniq -setword pantaskState INIT
     147
     148        # delete existing entries in the appropriate pantaskStates
     149        process_cleanup pzPendingImfile
    174150    end
    175151
    176152    task.exit     default
    177         echo "phaseZ pending exp: failure"
     153        showcommand failure
    178154    end
    179155    task.exit     timeout
    180         echo "phaseZ pending exp: timeout"
    181     end
    182 end
     156        showcommand timeout
     157    end
     158end
     159
    183160
    184161# get dsget to retreive an imfile and then call pztool -copydone
    … …  
    189166
    190167    task.exec
    191         queuesize pzPendingImfile -var N
    192         if ($N == 0) break
    193 
    194         # get a line out of the queue
    195         queuepop pzPendingImfile -var line -key 0 NEW
    196 
    197         # take the state out of the line
    198         strpop line state
    199 
    200         # put the line back into the queue with a stea of RUN
    201         queuepush pzPendingImfile -replace -key 1:2:3:7:8 "RUN $line"
    202 
    203         # the STATE has already been removed from the line
    204         list word -split $line
    205         $exp_id     = $word:0
    206         $camera     = $word:1
    207         $telescope  = $word:2
    208         $file_id    = $word:3
    209         $bytes      = $word:4
    210         $md5sum     = $word:5
    211         $class      = $word:6
    212         $class_id   = $word:7
    213         $uri        = $word:8
    214         $exp_tag    = $word:9
    215 
    216 #        # specify choice of remote host:(need to choose based on chips)
    217 #        if ($parallel)
    218 #            host anyhost
    219 #        else
    220 #            host local
    221 #        end
    222        
    223         # store the current line without the state
    224         options "$line"
    225 
    226         echo "parsing line $line"
    227         echo command dsget --uri $uri --filename $workdir/$exp_id.$class_id.fits --bytes $bytes --md5 $md5sum
    228         command dsget --uri $uri --filename $workdir/$exp_id.$class_id.fits --bytes $bytes --md5 $md5sum
     168        if ($network == 0) break
     169
     170        book npages pzPendingImfile -var N
     171        if ($N == 0)
     172            periods      -exec     20
     173            break
     174        end
     175        periods      -exec     0.2
     176
     177        # find an exp that needs imfiles fetched
     178        book getpage pzPendingImfile 0 -var pageName -key pantaskState INIT
     179        if ($pageName == NULL) break
     180
     181
     182        # set that exp to run
     183        book setword pzPendingImfile $pageName pantaskState RUN
     184
     185        book getword pzPendingImfile $pageName uri -var URI
     186        book getword pzPendingImfile $pageName exp_name -var EXP_NAME
     187        book getword pzPendingImfile $pageName class_id -var CLASS_ID
     188        book getword pzPendingImfile $pageName bytes -var BYTES
     189        book getword pzPendingImfile $pageName md5sum -var MD5SUM
     190
     191        # store the current page
     192        options $pageName
     193
     194        $run = dsget --uri $URI --filename $workdir/$EXP_NAME.$CLASS_ID.fits --bytes $BYTES --md5 $MD5SUM
     195
     196        # create the command line
     197        if ($VERBOSE > 1)
     198          echo command $run
     199        end
     200        command $run
    229201    end
    230202
    231203    # success
    232204    task.exit 0
    233         # could run copydone from here but how would the exit status be
    234         # handled?
    235 
    236         list word -split $options:0
    237         $exp_id     = $word:0
    238         $camera     = $word:1
    239         $telescope  = $word:2
    240         $file_id    = $word:3
    241         $bytes      = $word:4
    242         $md5sum     = $word:5
    243         $class      = $word:6
    244         $class_id   = $word:7
    245         $uri        = $word:8
    246         $exp_tag    = $word:9
    247         queuepush pzCopyDone -key 1:2:3:7:8 "NEW $options:0 $workdir/$exp_id.$class_id.fits"
    248         queuepush pzPendingExp -replace -key 1:2:3:7:8 "DONE $options:0"
     205        book getword options $pageName uri -var URI
     206        book getword options $pageName exp_name -var EXP_NAME
     207        book getword options $pageName class_id -var CLASS_ID
     208        book getword options $pageName bytes -var BYTES
     209        book getword options $pageName md5sum -var MD5SUM
     210
     211        book delpage pzPendingImfiles optoins
     212
     213        local $run = pztool -copydone -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -class $CLASS -class_id $CLASS_ID -uri  $workdir/$EXP_NAME.$CLASS_ID.fits
     214
     215        # create the command line
     216        if ($VERBOSE > 1)
     217          echo command $run
     218        end
     219        command $run
     220
     221        process_exit pzPendingExp $options:0 $JOB_STATUS
    249222    end
    250223
    251224    task.exit default
    252         queuepush pzPendingExp -replace -key 1:2:3:7:8 "FAIL $options:0"
     225        showcommand failure
     226        book setword pzPendingImfile $options:0 pantaskState $JOB_STATUS
    253227    end
    254228
    255229    task.exit timeout
    256         queuepush pzPendingExp -replace -key 1:2:3:7:8 "TIMEOUT $options:0"
    257     end
    258 end
    259 
    260 task pztool.copydone
    261     periods      -exec     5
    262     periods      -poll     5
    263     periods      -timeout  15
    264 
    265     host local
    266 
    267     task.exec
    268         queuesize pzCopyDone -var N
    269         if ($N == 0) break
    270 
    271         # get a line out of the queue
    272         queuepop pzCopyDone -var line -key 0 NEW
    273 
    274         # take the state out of the line
    275         strpop line state
    276 
    277         # put the line back into the queue with a stea of RUN
    278         queuepush pzCopyDone -replace -key 1:2:3:7:8 "RUN $line"
    279 
    280         # the STATE has already been removed from the line
    281         list word -split $line
    282         $exp_id     = $word:0
    283         $camera     = $word:1
    284         $telescope  = $word:2
    285         $file_id    = $word:3
    286         $bytes      = $word:4
    287         $md5sum     = $word:5
    288         $class      = $word:6
    289         $class_id   = $word:7
    290         $uri        = $word:8
    291         $exp_tag    = $word:9
    292         $new_uri    = $word:10
    293 
    294         # store the current line without the state
    295         options "$line"
    296 
    297         echo command pztool -copydone -exp_id $exp_id -inst $camera -telescope $telescope -class $class -class_id $class_id -uri $uri
    298         command pztool -copydone -exp_id $exp_id -inst $camera -telescope $telescope -class $class -class_id $class_id -uri $new_uri
    299     end
    300 
    301     # success
    302     task.exit 0
    303         queuepush pzCopyDone -replace -key 1:2:3:6:7 "DONE $options:0"
    304     end
    305 
    306     task.exit default
    307         queuepush pzCopyDone -replace -key 1:2:3:6:7 "FAIL $options:0"
    308     end
    309 
    310     task.exit timeout
    311         queuepush pzCopyDone -replace -key 1:2:3:6:7 "TIMEOUT $options:0"
    312     end
    313 end
     230        showcommand timeout
     231        book setword pzPendingImfile $options:0 pantaskState TIMEOUT
     232    end
     233end
Note: See TracChangeset for help on using the changeset viewer.