IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 26015


Ignore:
Timestamp:
Nov 3, 2009, 1:02:08 PM (17 years ago)
Author:
bills
Message:

Spread building of distribution bundles over a number of hosts (currently 5)

Location:
trunk
Files:
13 edited

Legend:

Unmodified
Added
Removed

  • trunk/dbconfig/changes.txt

    r25929 r26015  
    14241424ALTER TABLE publishClient ADD COLUMN active TINYINT DEFAULT 0 AFTER client_id;
    14251425
    1426 
    1427 
    1428 -- UPDATE dbversion set schema_version = '1.1.57',  updated= CURRENT_TIMESTAMP();
     1426# add outdir to allow spreading files over multiple hosts
     1427ALTER TABLE distRun ADD COLUMN outdir VARCHAR(255) AFTER outroot;
     1428ALTER TABLE distComponent ADD COLUMN outdir VARCHAR(255) AFTER state;
     1429
     1430# populate the existing runs (except for bills' test runs)
     1431UPDATE distRun SET outdir = CONCAT_WS('.', outroot, dist_id) WHERE outroot not like '%@HOST@%';
     1432UPDATE distRun join distComponent using(dist_id) SET distComponent.outdir = CONCAT_WS('.', distRun.outroot, dist_id);
     1433
     1434UPDATE dbversion set schema_version = '1.1.57',  updated= CURRENT_TIMESTAMP();
     1435
     1436-- Vesion 1.1.58
     1437-- UPDATE dbversion set schema_version = '1.1.58',  updated= CURRENT_TIMESTAMP();

  • trunk/dbconfig/dist.md

    r25835 r26015  
    1717    label       STR         64      # Key
    1818    outroot     STR         255
     19    outdir      STR         255
    1920    clean       BOOL        f
    2021    no_magic    BOOL        f
    … …  
    3132    md5sum      STR         32
    3233    state       STR         64      # Key
     34    outdir      STR         255
    3335    name        STR         255
    3436    fault       S16         0

  • trunk/ippScripts/scripts/dist_advancerun.pl

    r24433 r26015  
    181181{
    182182    my $command = "$disttool -updaterun -dist_id $dist_id -set_state full";
     183    $command .= " -set_outdir $outdir";
    183184    $command .= " -dbname $dbname" if defined $dbname;
    184185

  • trunk/ippScripts/scripts/dist_component.pl

    r25942 r26015  
    307307}
    308308{
    309     my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component";
     309    my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component -outdir $outdir";
    310310    $command .= " -name $file_name -bytes $bytes -md5sum $md5sum";
    311311    $command .= " -dbname $dbname" if defined $dbname;

  • trunk/ippScripts/scripts/dist_make_fileset.pl

    r25850 r26015  
    5353           'label=s'        => \$label,
    5454           'dist_group=s'   => \$dist_group,
    55            'filter=s'      => \$filter,
     55           'filter=s'       => \$filter,
    5656           'ds_dbhost=s'    => \$ds_dbhost,  # database host for the datastore database
    5757           'ds_dbname=s'    => \$ds_dbname,  # database name for the datastore database
    … …  
    9494
    9595# make sure that the database info file for this run exists
    96 my $dbinfo_file = "dbinfo.$stage.$stage_id.mdc";
    97 if (! -e "$dist_dir/$dbinfo_file" ) {
     96my $dbinfo_file = "$dist_dir/dbinfo.$stage.$stage_id.mdc";
     97if (! -e "$dbinfo_file" ) {
    9898    &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
    9999}
    … …  
    101101
    102102# make sure that the dirinfo file for this run exists
    103 my $dirinfo_file = "dirinfo.$stage.$stage_id.mdc";
    104 if (! -e "$dist_dir/$dirinfo_file" ) {
     103my $dirinfo_file = "$dist_dir/dirinfo.$stage.$stage_id.mdc";
     104if (! -e "$dirinfo_file" ) {
    105105    &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
    106106}
    … …  
    142142    my $md5sum = $component->{md5sum};
    143143    # name of the file
    144     my $name = $component->{name};
     144    my $file_name = $component->{name};
    145145    # component id (class_is or skycell_id)
     146    my $comp_dir = $component->{outdir};
    146147    my $comp_name = $component->{component};
    147148
    148149    # XXX: if tarfile is not always the right type we need to add a type to distComponent
    149     print $listFile "$name|$size|$md5sum|tgz|$comp_name|\n";
     150    print $listFile "$comp_dir/$file_name|$size|$md5sum|tgz|$comp_name|\n";
    150151}
    151152
    … …  
    156157
    157158    # the data store will refer to the distribution bundle via symlinks back to distRun.outdir
    158     $command .= " --datapath $dist_dir --link";
     159#    $command .= " --datapath $dist_dir --link";
     160    $command .= " --abspath --link";
    159161
    160162    # set the product specific columns in product list

  • trunk/ippTasks/dist.pro

    r25462 r26015  
    7070    active false
    7171  end
     72end
     73
     74macro set.dist.workdir.by.component
     75    # component template default varname
     76    if ($0 != 5)
     77        echo "USAGE: set.workdir.by.component (componentID) (template) (default) (varname)"
     78        break
     79    end
     80    local host component default template varname length start count component_id
     81    $component = $1
     82    $template = $2
     83    $default = $3
     84    $varname = $4
     85    if ("$template" == "NULL")
     86        echo "ERROR: WORKDIR template not set."
     87        break
     88    end
     89    book getword ipphosts distribution count -var count
     90    if ("$count" == "NULL")
     91        echo "ERROR: disthosts is empty"
     92        break
     93    end
     94    # take last two letters of component id
     95    # treat it as an integer and use modulous of length of distribution hosts
     96    # to compute a host
     97    strlen $component length
     98    $start = $length - 2
     99    substr $component $start 2 index
     100    $component_id = $index % $count
     101
     102    book getword ipphosts distribution $component_id -var host
     103    if ("$host" == "NULL")
     104        echo "ERROR: failed to find host for $component"
     105        $host = $default
     106    end
     107
     108    strsub $template @HOST@ $host -var $varname
    72109end
    73110
    … …  
    165202    book getword distToProcess $pageName no_magic -var NO_MAGIC
    166203    book getword distToProcess $pageName magicked -var MAGICKED
    167     book getword distToProcess $pageName outdir -var OUTDIR
     204    book getword distToProcess $pageName outdir -var OUTDIR_TEMPLATE
    168205    book getword distToProcess $pageName dbname -var DBNAME
    169206
    … …  
    187224#    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
    188225#    host anyhost
     226
     227    set.dist.workdir.by.component $COMPONENT $OUTDIR_TEMPLATE ipp049 OUTDIR
     228    if ("$OUTDIR" == "NULL")
     229        echo ERROR failed to set workdir for $COMPONENT
     230        break
     231    end
    189232
    190233    substr $COMPONENT 0 3 COMP_HEAD
    … …  
    294337    book getword distToAdvance $pageName stage   -var STAGE
    295338    book getword distToAdvance $pageName stage_id -var STAGE_ID
    296     book getword distToAdvance $pageName outdir -var OUTDIR
     339    book getword distToAdvance $pageName outdir -var OUTDIR_TEMPLATE
    297340    book getword distToAdvance $pageName clean -var CLEAN
    298341    book getword distToAdvance $pageName dbname -var DBNAME
    … …  
    304347    host anyhost
    305348
     349    # using $DIST_ID as the "component" works fine here since we only look
     350    # at the last two digits
     351    set.dist.workdir.by.component $DIST_ID $OUTDIR_TEMPLATE ipp049 OUTDIR
     352    if ("$OUTDIR" == "NULL")
     353        echo ERROR failed to set workdir for $DIST_ID
     354        break
     355    end
     356
    306357    sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID
    307358#    stdout $logfile

  • trunk/ippTasks/ipphosts.mhpcc.config

    r25839 r26015  
    214214  ota76  STR  ipp025
    215215END
     216
     217ipphosts METADATA
     218  camera STR distribution
     219  count  S32  5
     220  0      STR ipp048
     221  1      STR ipp049
     222  2      STR ipp050
     223  3      STR ipp051
     224  4      STR ipp052
     225END

  • trunk/ippTasks/rcserver.pro

    r25849 r26015  
    139139    book getword rcPendingFS $pageName dbname -var DBNAME
    140140
    141 #    set.host.for.camera $CAMERA $MAGIC_ID
    142 #    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
    143141    host anyhost
    144142

  • trunk/ippTools/share/disttool_pendingfileset.sql

    r25835 r26015  
    33    target_id,
    44    distRun.stage,
     5    distRun.outdir as dist_dir,
    56    stage_id,
    67    distTarget.dist_group,
    78    distTarget.filter,
    8     CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as dist_dir,
     9--    CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as dist_dir,
    910    rcDestination.name AS product_name,
    1011    rcDestination.dest_id,

  • trunk/ippTools/share/disttool_processedcomponent.sql

    r23842 r26015  
    55    stage_id,
    66    component,
    7     CONCAT_WS('.', outroot, CONVERT(dist_id, CHAR)) as outdir,
     7    distComponent.outdir,
    88    bytes,
    99    md5sum,

  • trunk/ippTools/share/pxadmin_create_tables.sql

    r25929 r26015  
    13441344    label       VARCHAR(64),
    13451345    outroot     VARCHAR(255),
     1346    outdir      VARCHAR(255),
    13461347    clean       TINYINT,
    13471348    no_magic    TINYINT,
    … …  
    13621363    md5sum      VARCHAR(32),
    13631364    state       VARCHAR(64),
     1365    outdir      VARCHAR(255),
    13641366    name        VARCHAR(255),
    13651367    fault       SMALLINT,

  • trunk/ippTools/src/disttool.c

    r26005 r26015  
    371371                new_label,
    372372                outroot,
     373                NULL,     // outdir
    373374                clean,
    374375                no_magic,
    375376                "new",
    376                 NULL,   // time_stamp
     377                NULL,    // time_stamp
    377378                0,       // fault
    378                 set_note    // note does not propagate
     379                set_note // note does not propagate
    379380                );
    380381
    … …  
    430431    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
    431432    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
     433    PXOPT_LOOKUP_STR(outdir, config->args, "-set_outdir", false, false);
    432434    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
    433435
    … …  
    440442    psString query = psStringCopy("UPDATE distRun SET time_stamp = UTC_TIMESTAMP()");
    441443
     444    if (outdir) {
     445        psStringAppend(&query, " , outdir = '%s'", outdir);
     446    }
    442447    if (state) {
    443448        psStringAppend(&query, " , state = '%s'", state);
    … …  
    652657    PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", require_fileinfo, false);
    653658    PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", require_fileinfo, false);
     659    PXOPT_LOOKUP_STR(outdir, config->args, "-outdir", require_fileinfo, false);
    654660    PXOPT_LOOKUP_STR(name, config->args, "-name", require_fileinfo, false);
    655661
    656     if (!distComponentInsert(config->dbh, dist_id, component, bytes, md5sum, "full", name, fault)) {
     662    if (!distComponentInsert(config->dbh, dist_id, component, bytes, md5sum, "full", outdir, name, fault)) {
    657663        psError(PS_ERR_UNKNOWN, false, "database error");
    658664        return false;

  • trunk/ippTools/src/disttoolConfig.c

    r25835 r26015  
    7070    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL);
    7171    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "new value for label", NULL);
     72    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_outdir", 0, "new value for outdir", NULL);
    7273    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage",     0, "value for stage", NULL);
    7374    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state",     0, "value for state", NULL);
    … …  
    9899    psMetadataAddS64(addprocessedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
    99100    psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component (required)", NULL);
     101    psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory", NULL);
    100102    psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL);
    101103    psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-bytes", 0, "define file size", 0);
Note: See TracChangeset for help on using the changeset viewer.