IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 16173


Ignore:
Timestamp:
Jan 20, 2008, 3:59:25 PM (18 years ago)
Author:
eugene
Message:

merging changes from eam_branch_20080109: adding workdir/volume mangling and chip/hostname relationships

Location:
trunk/ippTasks
Files:
12 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/camera.pro

    r14693 r16173  
    127127    book getword camPendingExp $pageName reduction -var REDUCTION
    128128
    129     # specify choice of remote host:(need to choose based on chips)
    130     if ($parallel)
    131       host anyhost
    132     else
    133       host local
    134     end
     129    # specify choice of remote host based on camera and chip (class_id)
     130    set.host.for.camera $camera FPA
    135131
    136132    # output log from filerule

  • trunk/ippTasks/chip.pro

    r14693 r16173  
    127127    book getword chipPendingImfile $pageName reduction -var REDUCTION
    128128
    129     # specify choice of remote host
    130     # XXX need to choose based on chips
    131     if ($parallel)
    132       host anyhost
    133     else
    134       host local
    135     end
     129    # specify choice of local or remote host based on camera and chip (class_id)
     130    set.host.for.camera $camera $class_id
    136131
    137132    ## generate output log based on filerule

  • trunk/ippTasks/chiphosts.config

    r14063 r16173  
    22chiphosts MULTI
    33
     4# these camera names must match those listed in ipprc.config
    45chiphosts METADATA
    5   camera STR megacam
     6  camera STR MEGACAM
    67  ccd00 STR po00
    78  ccd01 STR po01
    … …  
    1718
    1819chiphosts METADATA
    19   camera STR cfh12k
     20  camera STR CFH12K
    2021  ccd00 STR po10
    2122  ccd01 STR po11
    2223  ccd02 STR po12
    2324END
     25
     26$gpcHost00 = ipp005
     27$gpcHost01 = ipp008
     28$gpcHost02 = ipp019
     29$gpcHost03 = ipp020
     30$gpcHost04 = ipp021
     31
     32chiphosts METADATA
     33  camera GPC1
     34  XY01  STR  gpcHost00
     35  XY02  STR  gpcHost00
     36  XY03  STR  gpcHost00
     37  XY04  STR  gpcHost00
     38  XY05  STR  gpcHost00
     39  XY06  STR  gpcHost00
     40  XY10  STR  gpcHost00
     41  XY11  STR  gpcHost00
     42  XY12  STR  gpcHost00
     43  XY13  STR  gpcHost00
     44  XY14  STR  gpcHost00
     45  XY15  STR  gpcHost00
     46
     47  XY16  STR  gpcHost01
     48  XY17  STR  gpcHost01
     49  XY20  STR  gpcHost01
     50  XY21  STR  gpcHost01
     51  XY22  STR  gpcHost01
     52  XY23  STR  gpcHost01
     53  XY24  STR  gpcHost01
     54  XY25  STR  gpcHost01
     55  XY26  STR  gpcHost01
     56  XY27  STR  gpcHost01
     57  XY30  STR  gpcHost01
     58  XY31  STR  gpcHost01
     59
     60  XY32  STR  gpcHost02
     61  XY33  STR  gpcHost02
     62  XY34  STR  gpcHost02
     63  XY35  STR  gpcHost02
     64  XY36  STR  gpcHost02
     65  XY37  STR  gpcHost02
     66  XY40  STR  gpcHost02
     67  XY41  STR  gpcHost02
     68  XY42  STR  gpcHost02
     69  XY43  STR  gpcHost02
     70  XY44  STR  gpcHost02
     71  XY45  STR  gpcHost02
     72
     73  XY46  STR  gpcHost03
     74  XY47  STR  gpcHost03
     75  XY50  STR  gpcHost03
     76  XY51  STR  gpcHost03
     77  XY52  STR  gpcHost03
     78  XY53  STR  gpcHost03
     79  XY54  STR  gpcHost03
     80  XY55  STR  gpcHost03
     81  XY56  STR  gpcHost03
     82  XY57  STR  gpcHost03
     83  XY60  STR  gpcHost03
     84  XY61  STR  gpcHost03
     85
     86  XY62  STR  gpcHost04
     87  XY63  STR  gpcHost04
     88  XY64  STR  gpcHost04
     89  XY65  STR  gpcHost04
     90  XY66  STR  gpcHost04
     91  XY67  STR  gpcHost04
     92  XY71  STR  gpcHost04
     93  XY72  STR  gpcHost04
     94  XY73  STR  gpcHost04
     95  XY74  STR  gpcHost04
     96  XY75  STR  gpcHost04
     97  XY76  STR  gpcHost04
     98END

  • trunk/ippTasks/detrend.correct.pro

    r15313 r16173  
    148148    book getword detPendingCorrectedImfile $pageName dbname    -var DBNAME
    149149
    150     # specify choice of remote host:
    151     if ($parallel)
    152       host anyhost
    153     else
    154       host local
    155     end
     150    # specify choice of local or remote host based on camera and chip (class_id)
     151    set.host.for.camera $camera $class_id
    156152
    157153    ## output log filename
    … …  
    279275    book getword detPendingCorrectedExp $pageName dbname    -var DBNAME
    280276
    281     # specify choice of remote host:
    282     if ($parallel)
    283       host anyhost
    284     else
    285       host local
    286     end
     277    # specify choice of local or remote host based on camera and chip (class_id)
     278    set.host.for.camera $camera FPA
    287279
    288280    # XXX push this into the filerules?

  • trunk/ippTasks/detrend.norm.pro

    r15854 r16173  
    156156    book getword detPendingNormStatImfile $pageName dbname    -var DBNAME
    157157
    158     # specify choice of remote host:
    159     if ($parallel)
    160       host anyhost
    161     else
    162       host local
    163     end
     158    # specify choice of local or remote host based on camera and chip (class_id)
     159    set.host.for.camera $camera FPA
    164160
    165161    if ("$WORKDIR" == "NULL")
    … …  
    278274    book getword detPendingNormImfile $pageName dbname    -var DBNAME
    279275
    280     # specify choice of remote host:
    281     if ($parallel)
    282       host anyhost
    283     else
    284       host local
    285     end
     276    # specify choice of local or remote host based on camera and chip (class_id)
     277    set.host.for.camera $camera $class_id
    286278
    287279    # XXX use ipp_filename.pl to lookup output file names
    … …  
    398390    book getword detPendingNormExp $pageName dbname    -var DBNAME
    399391
    400     # specify choice of remote host:
    401     if ($parallel)
    402       host anyhost
    403     else
    404       host local
    405     end
     392    # specify choice of local or remote host based on camera and chip (class_id)
     393    set.host.for.camera $camera FPA
    406394
    407395    # XXX add $WORKDIR/$LOG_DIR

  • trunk/ippTasks/detrend.process.pro

    r14693 r16173  
    147147    book getword detPendingProcessedImfile $pageName reduction -var REDUCTION
    148148
    149     # specify choice of remote host:
    150     if ($parallel)
    151       host anyhost
    152     else
    153       host local
    154     end
     149    # specify choice of local or remote host based on camera and chip (class_id)
     150    set.host.for.camera $camera $class_id
    155151
    156152    ## output log filename
    … …  
    276272    book getword detPendingProcessedExp $pageName dbname    -var DBNAME
    277273
    278     # specify choice of remote host:
    279     if ($parallel)
    280       host anyhost
    281     else
    282       host local
    283     end
     274    # specify choice of local or remote host based on camera and chip (class_id)
     275    set.host.for.camera $camera FPA
    284276
    285277    # XXX push this into the filerules?

  • trunk/ippTasks/detrend.reject.pro

    r14693 r16173  
    125125    book getword detRejectExp $pageName dbname    -var DBNAME
    126126
    127     # specify choice of remote host:
    128     if ($parallel)
    129       host anyhost
    130     else
    131       host local
    132     end
     127    # specify choice of local or remote host based on camera and chip (class_id)
     128    set.host.for.camera $camera FPA
    133129
    134130    if ("$WORKDIR" == "NULL")

  • trunk/ippTasks/detrend.resid.pro

    r14693 r16173  
    147147    book getword detPendingResidImfile $pageName reduction -var REDUCTION
    148148
    149     # specify choice of remote host:
    150     if ($parallel)
    151       host anyhost
    152     else
    153       host local
    154     end
     149    # specify choice of local or remote host based on camera and chip (class_id)
     150    set.host.for.camera $camera $class_id
    155151
    156152    # XXX use ipp_filename.pl to lookup output file names
    … …  
    279275    book getword detPendingResidExp $pageName dbname    -var DBNAME
    280276
    281     # specify choice of remote host:
    282     if ($parallel)
    283       host anyhost
    284     else
    285       host local
    286     end
     277    # specify choice of local or remote host based on camera and chip (class_id)
     278    set.host.for.camera $camera FPA
    287279
    288280    # XXX use ipp_filename.pl to lookup output file names

  • trunk/ippTasks/detrend.stack.pro

    r14693 r16173  
    127127    book getword detPendingStackedImfile $pageName reduction -var REDUCTION
    128128
    129     # specify choice of remote host:
    130     if ($parallel)
    131       host anyhost
    132     else
    133       host local
    134     end
     129    # specify choice of local or remote host based on camera and chip (class_id)
     130    set.host.for.camera $camera $class_id
    135131
    136132    if ("$WORKDIR" == "NULL")

  • trunk/ippTasks/notes.txt

    r15313 r16173  
     1
     22008.01.18
     3
     4  I'm working on the summit.copy.pro interaction with nebulous (and
     5  those of the other ippTasks) and the assignment of jobs and data
     6  locations to hosts on the basis of the chip/host relationship.  I
     7  have defined a table which relates the chip (class_id) to the
     8  desired host name.  There are a few places, and a few slightly
     9  different ways, in which this gets used:
     10
     11  * choice of host for command execution:
     12
     13    every job which operates on a specific chip must be told the chip
     14    host so it can run the job on that machine.  the information is
     15    supplied in pantasks to the job with the 'host' command.  I've
     16    made a pantasks macro 'set.host.by.camera' which takes the camera
     17    and class_id, looks up the match in the chip/host table, and calls
     18    the host command with the desired host.  If there is no match
     19    (camera not listed in chip/host table or chip not listed in
     20    table), the host is set to 'anyhost', which randomly chooses one.
     21    I have used this command for all tasks which generate a pcontrol
     22    command.  For the tasks which are performed at the camera level,
     23    these are set to anyhost.  For the tasks which operate on stacks,
     24    we can use the same structure to associate the skycell with a
     25    host. 
     26
     27  * dsget output target:
     28
     29    dsget is called by pantasks in the task 'dsget'.  currently, dsget
     30    accepts a --filename argument which specifies the output filename
     31    to use.  we have to tell t
     32 
     33
     34  * choice of output directory.   
     35
     36    the
     37
    138
    2392007.10.11

  • trunk/ippTasks/pantasks.pro

    r15780 r16173  
    11
    22# XXX make these upper-case
     3$NEBULOUS = 0
    34$network = 1
    45$parallel = 1
    … …  
    339340  ??
    340341end
     342
     343macro set.workdir.by.camera
     344  if ($0 != 3)
     345    echo "USAGE: set.workdir.by.camera (camera) (class_id)"
     346    break
     347  end
     348
     349  # missing camera and/or chiphosts table results in host = NULL
     350  book getword chiphosts $1 $2 -var $host
     351
     352  if ($NEBULOUS)
     353    if ("$host" == "NULL")
     354      $workdir = $workdir_default
     355      $volume  = "NULL"
     356      return
     357    end
     358    strsub $workdir_template @HOST@ $host -var workdir
     359    strsub $volume_template @HOST@ $host -var volume
     360    return
     361  end
     362
     363  if ("$host" == "NULL")
     364    $workdir = $workdir_default
     365    return
     366  end
     367  strsub $workdir_template @HOST@ $host -var workdir
     368end
     369
     370macro set.host.by.camera
     371  if ($0 != 3)
     372    echo "USAGE: set.host.by.camera (camera) (class_id)"
     373    break
     374  end
     375
     376  if (not($parallel))
     377    host local
     378    return
     379  end
     380
     381  # missing camera and/or chiphosts table results in host = NULL
     382  book getword chiphosts $1 $2 -var $host
     383
     384  if ("$host" == "NULL")
     385    host anyhost
     386  else
     387    host $host
     388  end
     389end

  • trunk/ippTasks/summit.copy.pro

    r15298 r16173  
    77# pztool -adddatastore -inst allskycam -telescope ps1 -uri  http://otis1.ifa.hawaii.edu/ds/allskycam/index.txt
    88
    9 $workdir    = /data/ipp010.0/
     9# the templates are used if we have a class_id/host relationship; if none is found, the default values are used
     10# XXX not sure how to handle the .N value if we need to use more than one
     11if ($NEBULOUS)
     12 $workdir_default  = neb://
     13 $workdir_template = neb://
     14 $volume_default  = ipp004-v0
     15 $volume_template = @HOST@-v0
     16else
     17 $workdir_default  = /data/ipp004.0/
     18 $workdir_template = /data/@HOST@.0/
     19end
    1020
    1121$VERBOSE = 1
    … …  
    3444    periods      -poll     1
    3545    periods      -timeout  15
     46    npending     1
    3647
    3748    # success
    … …  
    231242    periods      -poll     0.05
    232243    periods      -timeout  120
    233     host        anyhost
    234244
    235245    task.exec
    … …  
    264274        substr $DATEOBS 8 2 DAY
    265275
     276        # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship
     277        # this function uses workdir_template, workdir_default, volume_template, volume_default,
     278        # it sets workdir and volume
     279        set.workdir.by.camera $CAMERA $CLASS_ID
     280
    266281        # figure out filename
     282        # XXXX this filename needs to use nebulous names
    267283        $FILENAME = $workdir/$CAMERA/$YEAR/$MONTH/$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits
     284        $FILEROOT = $workdir/$CAMERA/$YEAR/$MONTH/$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID
    268285
    269286        book setword pzPendingImfile $pageName filename $FILENAME
     287
     288        set.host.by.camera $CAMERA $CLASS_ID
     289        host        anyhost
    270290
    271291        # store the pageName with this job
    272292        options $pageName
    273 
    274         if ($MD5SUM == 0)
    275             $run = dsget --uri $URI --filename $FILENAME --bytes $BYTES
    276         else
    277             $run = dsget --uri $URI --filename $FILENAME --bytes $BYTES --md5 $MD5SUM
     293        options $FILEROOT
     294
     295        $run = dsget --uri $URI --filename $FILENAME --bytes $BYTES
     296        if ($MD5SUM)
     297            $run = $run --md5 $MD5SUM
     298        end
     299        if ($NEBULOUS && ("$volname" != "NULL"))
     300            $run = $run --volume $volume
    278301        end
    279302
    … …  
    295318        book getword pzPendingImfile $options:0 filename -var FILENAME
    296319
     320        # we need to set the values of tess_id, dvodb, end_stage, workdir here
     321        # these are functions of (at least) EXP_TYPE and SURVEY_ID, both of which should come from the data store
     322
    297323        # update the database with the success
    298         $run = pztool -copydone -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -class $CLASS -class_id $CLASS_ID -uri $FILENAME
     324        $run = pztool -copydone -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -class $CLASS -class_id $CLASS_ID -uri $FILENAME -workdir $options:1
     325        # tess_id, dvodb, workdir, end_stage
    299326
    300327        # create the command line
Note: See TracChangeset for help on using the changeset viewer.