Index: /trunk/dbconfig/changes.txt =================================================================== --- /trunk/dbconfig/changes.txt (revision 26014) +++ /trunk/dbconfig/changes.txt (revision 26015) @@ -1424,5 +1424,14 @@ ALTER TABLE publishClient ADD COLUMN active TINYINT DEFAULT 0 AFTER client_id; - - --- UPDATE dbversion set schema_version = '1.1.57', updated= CURRENT_TIMESTAMP(); +# add outdir to allow spreading files over multiple hosts +ALTER TABLE distRun ADD COLUMN outdir VARCHAR(255) AFTER outroot; +ALTER TABLE distComponent ADD COLUMN outdir VARCHAR(255) AFTER state; + +# populate the existing runs (except for bills' test runs) +UPDATE distRun SET outdir = CONCAT_WS('.', outroot, dist_id) WHERE outroot not like '%@HOST@%'; +UPDATE distRun join distComponent using(dist_id) SET distComponent.outdir = CONCAT_WS('.', distRun.outroot, dist_id); + +UPDATE dbversion set schema_version = '1.1.57', updated= CURRENT_TIMESTAMP(); + +-- Vesion 1.1.58 +-- UPDATE dbversion set schema_version = '1.1.58', updated= CURRENT_TIMESTAMP(); Index: /trunk/dbconfig/dist.md =================================================================== --- /trunk/dbconfig/dist.md (revision 26014) +++ /trunk/dbconfig/dist.md (revision 26015) @@ -17,4 +17,5 @@ label STR 64 # Key outroot STR 255 + outdir STR 255 clean BOOL f no_magic BOOL f @@ -31,4 +32,5 @@ md5sum STR 32 state STR 64 # Key + outdir STR 255 name STR 255 fault S16 0 Index: /trunk/ippScripts/scripts/dist_advancerun.pl =================================================================== --- /trunk/ippScripts/scripts/dist_advancerun.pl (revision 26014) +++ /trunk/ippScripts/scripts/dist_advancerun.pl (revision 26015) @@ -181,4 +181,5 @@ { my $command = "$disttool -updaterun -dist_id $dist_id -set_state full"; + $command .= " -set_outdir $outdir"; $command .= " -dbname $dbname" if defined $dbname; Index: /trunk/ippScripts/scripts/dist_component.pl =================================================================== --- /trunk/ippScripts/scripts/dist_component.pl (revision 26014) +++ /trunk/ippScripts/scripts/dist_component.pl (revision 26015) @@ -307,5 +307,5 @@ } { - my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component"; + my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component -outdir $outdir"; $command .= " -name $file_name -bytes $bytes -md5sum $md5sum"; $command .= " -dbname $dbname" if defined $dbname; Index: /trunk/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- /trunk/ippScripts/scripts/dist_make_fileset.pl (revision 26014) +++ /trunk/ippScripts/scripts/dist_make_fileset.pl (revision 26015) @@ -53,5 +53,5 @@ 'label=s' => \$label, 'dist_group=s' => \$dist_group, - 'filter=s' => \$filter, + 'filter=s' => \$filter, 'ds_dbhost=s' => \$ds_dbhost, # database host for the datastore database 'ds_dbname=s' => \$ds_dbname, # database name for the datastore database @@ -94,6 +94,6 @@ # make sure that the database info file for this run exists -my $dbinfo_file = "dbinfo.$stage.$stage_id.mdc"; -if (! -e "$dist_dir/$dbinfo_file" ) { +my $dbinfo_file = "$dist_dir/dbinfo.$stage.$stage_id.mdc"; +if (! -e "$dbinfo_file" ) { &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -101,6 +101,6 @@ # make sure that the dirinfo file for this run exists -my $dirinfo_file = "dirinfo.$stage.$stage_id.mdc"; -if (! -e "$dist_dir/$dirinfo_file" ) { +my $dirinfo_file = "$dist_dir/dirinfo.$stage.$stage_id.mdc"; +if (! -e "$dirinfo_file" ) { &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -142,10 +142,11 @@ my $md5sum = $component->{md5sum}; # name of the file - my $name = $component->{name}; + my $file_name = $component->{name}; # component id (class_is or skycell_id) + my $comp_dir = $component->{outdir}; my $comp_name = $component->{component}; # XXX: if tarfile is not always the right type we need to add a type to distComponent - print $listFile "$name|$size|$md5sum|tgz|$comp_name|\n"; + print $listFile "$comp_dir/$file_name|$size|$md5sum|tgz|$comp_name|\n"; } @@ -156,5 +157,6 @@ # the data store will refer to the distribution bundle via symlinks back to distRun.outdir - $command .= " --datapath $dist_dir --link"; +# $command .= " --datapath $dist_dir --link"; + $command .= " --abspath --link"; # set the product specific columns in product list Index: /trunk/ippTasks/dist.pro =================================================================== --- /trunk/ippTasks/dist.pro (revision 26014) +++ /trunk/ippTasks/dist.pro (revision 26015) @@ -70,4 +70,41 @@ active false end +end + +macro set.dist.workdir.by.component + # component template default varname + if ($0 != 5) + echo "USAGE: set.workdir.by.component (componentID) (template) (default) (varname)" + break + end + local host component default template varname length start count component_id + $component = $1 + $template = $2 + $default = $3 + $varname = $4 + if ("$template" == "NULL") + echo "ERROR: WORKDIR template not set." + break + end + book getword ipphosts distribution count -var count + if ("$count" == "NULL") + echo "ERROR: disthosts is empty" + break + end + # take last two letters of component id + # treat it as an integer and use modulous of length of distribution hosts + # to compute a host + strlen $component length + $start = $length - 2 + substr $component $start 2 index + $component_id = $index % $count + + book getword ipphosts distribution $component_id -var host + if ("$host" == "NULL") + echo "ERROR: failed to find host for $component" + $host = $default + end + + strsub $template @HOST@ $host -var $varname end @@ -165,5 +202,5 @@ book getword distToProcess $pageName no_magic -var NO_MAGIC book getword distToProcess $pageName magicked -var MAGICKED - book getword distToProcess $pageName outdir -var OUTDIR + book getword distToProcess $pageName outdir -var OUTDIR_TEMPLATE book getword distToProcess $pageName dbname -var DBNAME @@ -187,4 +224,10 @@ # set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR # host anyhost + + set.dist.workdir.by.component $COMPONENT $OUTDIR_TEMPLATE ipp049 OUTDIR + if ("$OUTDIR" == "NULL") + echo ERROR failed to set workdir for $COMPONENT + break + end substr $COMPONENT 0 3 COMP_HEAD @@ -294,5 +337,5 @@ book getword distToAdvance $pageName stage -var STAGE book getword distToAdvance $pageName stage_id -var STAGE_ID - book getword distToAdvance $pageName outdir -var OUTDIR + book getword distToAdvance $pageName outdir -var OUTDIR_TEMPLATE book getword distToAdvance $pageName clean -var CLEAN book getword distToAdvance $pageName dbname -var DBNAME @@ -304,4 +347,12 @@ host anyhost + # using $DIST_ID as the "component" works fine here since we only look + # at the last two digits + set.dist.workdir.by.component $DIST_ID $OUTDIR_TEMPLATE ipp049 OUTDIR + if ("$OUTDIR" == "NULL") + echo ERROR failed to set workdir for $DIST_ID + break + end + sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID # stdout $logfile Index: /trunk/ippTasks/ipphosts.mhpcc.config =================================================================== --- /trunk/ippTasks/ipphosts.mhpcc.config (revision 26014) +++ /trunk/ippTasks/ipphosts.mhpcc.config (revision 26015) @@ -214,2 +214,12 @@ ota76 STR ipp025 END + +ipphosts METADATA + camera STR distribution + count S32 5 + 0 STR ipp048 + 1 STR ipp049 + 2 STR ipp050 + 3 STR ipp051 + 4 STR ipp052 +END Index: /trunk/ippTasks/rcserver.pro =================================================================== --- /trunk/ippTasks/rcserver.pro (revision 26014) +++ /trunk/ippTasks/rcserver.pro (revision 26015) @@ -139,6 +139,4 @@ book getword rcPendingFS $pageName dbname -var DBNAME -# set.host.for.camera $CAMERA $MAGIC_ID -# set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR host anyhost Index: /trunk/ippTools/share/disttool_pendingfileset.sql =================================================================== --- /trunk/ippTools/share/disttool_pendingfileset.sql (revision 26014) +++ /trunk/ippTools/share/disttool_pendingfileset.sql (revision 26015) @@ -3,8 +3,9 @@ target_id, distRun.stage, + distRun.outdir as dist_dir, stage_id, distTarget.dist_group, distTarget.filter, - CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as dist_dir, +-- CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as dist_dir, rcDestination.name AS product_name, rcDestination.dest_id, Index: /trunk/ippTools/share/disttool_processedcomponent.sql =================================================================== --- /trunk/ippTools/share/disttool_processedcomponent.sql (revision 26014) +++ /trunk/ippTools/share/disttool_processedcomponent.sql (revision 26015) @@ -5,5 +5,5 @@ stage_id, component, - CONCAT_WS('.', outroot, CONVERT(dist_id, CHAR)) as outdir, + distComponent.outdir, bytes, md5sum, Index: /trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_create_tables.sql (revision 26014) +++ /trunk/ippTools/share/pxadmin_create_tables.sql (revision 26015) @@ -1344,4 +1344,5 @@ label VARCHAR(64), outroot VARCHAR(255), + outdir VARCHAR(255), clean TINYINT, no_magic TINYINT, @@ -1362,4 +1363,5 @@ md5sum VARCHAR(32), state VARCHAR(64), + outdir VARCHAR(255), name VARCHAR(255), fault SMALLINT, Index: /trunk/ippTools/src/disttool.c =================================================================== --- /trunk/ippTools/src/disttool.c (revision 26014) +++ /trunk/ippTools/src/disttool.c (revision 26015) @@ -371,10 +371,11 @@ new_label, outroot, + NULL, // outdir clean, no_magic, "new", - NULL, // time_stamp + NULL, // time_stamp 0, // fault - set_note // note does not propagate + set_note // note does not propagate ); @@ -430,4 +431,5 @@ PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-set_outdir", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -440,4 +442,7 @@ psString query = psStringCopy("UPDATE distRun SET time_stamp = UTC_TIMESTAMP()"); + if (outdir) { + psStringAppend(&query, " , outdir = '%s'", outdir); + } if (state) { psStringAppend(&query, " , state = '%s'", state); @@ -652,7 +657,8 @@ PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", require_fileinfo, false); PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", require_fileinfo, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-outdir", require_fileinfo, false); PXOPT_LOOKUP_STR(name, config->args, "-name", require_fileinfo, false); - if (!distComponentInsert(config->dbh, dist_id, component, bytes, md5sum, "full", name, fault)) { + if (!distComponentInsert(config->dbh, dist_id, component, bytes, md5sum, "full", outdir, name, fault)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; Index: /trunk/ippTools/src/disttoolConfig.c =================================================================== --- /trunk/ippTools/src/disttoolConfig.c (revision 26014) +++ /trunk/ippTools/src/disttoolConfig.c (revision 26015) @@ -70,4 +70,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_outdir", 0, "new value for outdir", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage", 0, "value for stage", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "value for state", NULL); @@ -98,4 +99,5 @@ psMetadataAddS64(addprocessedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component (required)", NULL); + psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory", NULL); psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-bytes", 0, "define file size", 0);