Changeset 23861

Timestamp:

Apr 14, 2009, 3:14:27 PM (17 years ago)

Author:

bills

Message:

add task to build data store filesets for distRuns

Location:

trunk

Files:

• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/scripts/dist_advancerun.pl (modified) (1 diff)
• ippScripts/scripts/dist_make_fileset.pl (modified) (6 diffs)
• ippTasks/Makefile.am (modified) (1 diff)
• ippTasks/rcserver.pro (added)
• ippTools/share/disttool_pendingfileset.sql (modified) (1 diff)
• ippTools/src/disttool.c (modified) (3 diffs)
• ippTools/src/disttool.h (modified) (1 diff)
• ippTools/src/disttoolConfig.c (modified) (2 diffs)

trunk/ippScripts/Build.PL

@@ -82 +82 @@
         scripts/dist_component.pl
         scripts/dist_advancerun.pl
+        scripts/dist_make_fileset.pl
     )],
     dist_abstract => 'Scripts for running the Pan-STARRS IPP',

trunk/ippScripts/scripts/dist_advancerun.pl

@@ -89 +89 @@
 }
 
-# XXX we should create a file rule for this
+# XXX should we create a file rule for this?
 my $outfile = "$outdir/dbinfo.$stage.$stage_id.mdc";
 

trunk/ippScripts/scripts/dist_make_fileset.pl

@@ -39 +39 @@
 
 # Parse the command-line arguments
-my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $prod_id, $product_name, $product_root, $ds_dbhost, $ds_dbname);
+my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $prod_id, $product_name, $ds_dbhost, $ds_dbname);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 
@@ -50 +50 @@
            'prod_id=s'      => \$prod_id,    # id for the product
            'product_name=s' => \$product_name,  # location of the data store directory for this product
-           'product_root=s' => \$product_root,  # location of the data store directory for this product
            'ds_dbhost=s'    => \$ds_dbhost,  # database host for the datastore database
            'ds_dbname=s'    => \$ds_dbname,  # database name for the datastore database
@@ -61 +60 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --prod_id --product_root --ds_dbhost --ds_dbname",
+pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --prod_id --ds_dbhost --ds_dbname",
            -exitval => 3) unless
     defined $dist_id and
@@ -70 +69 @@
     defined $prod_id and
     defined $product_name and
-    defined $product_root and
     defined $ds_dbhost and
     defined $ds_dbname;
@@ -156 +154 @@
 
 # XXX: disttool -addrcdsfileset
-
-# XXX todo: add rcRun's for the destinations
+{
+    my $command = "$disttool -addfileset -dist_id $dist_id -prod_id $prod_id -name $fileset_name";
+    $command .= " -dbname $dbname" if $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code);
+    }
+}
+
 
 
@@ -239 +247 @@
     print STDERR "$msg\n";
 
+    my $command = "$disttool -addfileset -dist_id $dist_id -prod_id $prod_id -fault $fault";
+    $command .= " -dbname $dbname" if $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        print STDERR "Unable to perform $command error_code: $error_code\n";
+    }
     exit $fault;
 }

trunk/ippTasks/Makefile.am

@@ -21 +21 @@
         replicate.pro \
         dist.pro \
+        rcserver.pro \
         pstamp.pro
 

trunk/ippTools/share/disttool_pendingfileset.sql

@@ -5 +5 @@
     stage_id,
     CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as dist_dir,
-    rcDSProduct.*
+    rcDSProduct.name AS product_name,
+    rcDSProduct.prod_id,
+    rcDSProduct.dbname AS ds_dbname,
+    rcDSProduct.dbhost AS ds_dbhost
 FROM rcDestination 
 JOIN rcInterest USING(dest_id) 

trunk/ippTools/src/disttool.c

@@ -41 +41 @@
 static bool pendingfilesetMode(pxConfig *config);
 static bool addfilesetMode(pxConfig *config);
+static bool pendingdatastoresMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -71 +72 @@
         MODECASE(DISTTOOL_MODE_PENDINGFILESET, pendingfilesetMode);
         MODECASE(DISTTOOL_MODE_ADDFILESET, addfilesetMode);
+        MODECASE(DISTTOOL_MODE_PENDINGDATASTORES, pendingdatastoresMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -893 +895 @@
 }
 
+static bool pendingdatastoresMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+#ifdef notyet
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // look for "inputs" that need to processed
+    psString query = pxDataGet("disttool_pendingfileset.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("disttool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "pendingfileset", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+# endif // notdef
+    return true;
+}

trunk/ippTools/src/disttool.h

@@ -35 +35 @@
     DISTTOOL_MODE_PENDINGFILESET,
     DISTTOOL_MODE_ADDFILESET,
+    DISTTOOL_MODE_PENDINGDATASTORES,
 } disttoolMode;
 

trunk/ippTools/src/disttoolConfig.c

@@ -138 +138 @@
     psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL);
     psMetadataAddS32(addfilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
+    //
+    // -pendingfileset
+    psMetadata *pendingdatastoresArgs = psMetadataAlloc();
+    psMetadataAddS64(pendingdatastoresArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
+    psMetadataAddStr(pendingdatastoresArgs, PS_LIST_TAIL, "-label",    0, "limit results to label", NULL);
+    psMetadataAddStr(pendingdatastoresArgs, PS_LIST_TAIL, "-stage",    0, "limit results to runs for stage", NULL);
+    psMetadataAddU64(pendingdatastoresArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
+    psMetadataAddBool(pendingdatastoresArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
     psMetadata *argSets = psMetadataAlloc();
     psMetadata *modes = psMetadataAlloc();
@@ -151 +160 @@
     PXOPT_ADD_MODE("-pendingfileset",      "", DISTTOOL_MODE_PENDINGFILESET, pendingfilesetArgs);
     PXOPT_ADD_MODE("-addfileset",      "", DISTTOOL_MODE_ADDFILESET, addfilesetArgs);
+    PXOPT_ADD_MODE("-pendingdatastores",      "", DISTTOOL_MODE_PENDINGDATASTORES, pendingdatastoresArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {