IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 23894


Ignore:
Timestamp:
Apr 16, 2009, 5:54:55 PM (17 years ago)
Author:
Paul Price
Message:

Distribution client workflow is now working!

Location:
trunk
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/receive_file.pl

    r23890 r23894  
    6565my $filename = "$tempdir/$file";  # Target filename
    6666{
    67     my $uri = "$source/product/$fileset/$file"; # URI for datastore file
     67    my $uri = "$source/$product/$fileset/$file"; # URI for datastore file
    6868    my $command = "dsget --uri $uri --filename $filename"; # Command to execute
    6969
    … …  
    8181
    8282    my $command = "$tool -importrun -infile $filename"; # Command to execute
     83    $command .= " -dbname $dbname" if defined $dbname;
    8384    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    8485        run(command => $command, verbose => $verbose);
    … …  
    8889    my @files = ();
    8990    {
    90         my $command = "tar -C $tempdir -tvzf $filename"; # Command to execute
     91        my $command = "tar -C $tempdir -tzf $filename"; # Command to execute
    9192        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    9293            run(command => $command, verbose => $verbose);
    9394        die "Unable to get listing of tar file $filename\n" unless $success;
    9495
    95         foreach my $line (@$stdout_buf) {
     96        my @lines = split(/\n/, join "", @$stdout_buf); # Lines from output
     97        foreach my $line ( @lines ) {
    9698            $line =~ s/\#.*$//;
    9799            next unless $line =~ /\S+/;
    … …  
    99101            my $file = $fields[0];      # Name of file
    100102            $file =~ s|^./||;
    101             push @files, $file if defined $file;
     103            push @files, $file if $file =~ /\S+/;
    102104        }
    103105    }
    … …  
    115117        my $from = "$tempdir/$file"; # Source for file
    116118        my $target = "$workdir/$file"; # Target destination for file
    117         my $to = $ipprc->file_prepare( $target ); # Target for move
     119        my $to = $ipprc->file_create( $target ); # Target for move
    118120        system("mv $from $to") == 0 or die "Unable to move $file into workdir $workdir: $!\n";
    119121    }
    … …  
    129131{
    130132    my $command = "$receivetool -addresult";
    131     $command .= " -file $file_id";
     133    $command .= " -file_id $file_id";
    132134    $command .= (" -dtime_copy " . (($mjd_copy - $mjd_start) * 86400));
    133     $command .= (" -dtime_extract " . (($mjd_extract - $mjd_start) * 86400));
     135    $command .= (" -dtime_extract " . (($mjd_extract - $mjd_copy) * 86400));
    134136    $command .= " -dbname $dbname" if defined $dbname;
     137
     138
     139    print "$command\n";
    135140
    136141    unless (defined $no_update) {

  • trunk/ippScripts/scripts/receive_fileset.pl

    r23888 r23894  
    5555my @files = ();                 # Files to add
    5656{
    57     my $uri = "$source/product/$fileset"; # URI for datastore fileset
     57    my $uri = "$source/$product/$fileset"; # URI for datastore fileset
    5858    $uri .= "/index.txt" unless $uri =~ m|/index.txt$|;
    5959    my $command = "dsfilesetls --uri $uri"; # Command to execute
    … …  
    6464
    6565    # Get files
    66     foreach my $line (@$stdout_buf) {
     66    my @lines = split(/\n/, join "", @$stdout_buf); # Lines from output
     67    foreach my $line ( @lines ) {
    6768        $line =~ s/\#.*$//;
    6869        next unless $line =~ /\S+/;
    … …  
    7778    my $command = "receivetool -addfile"; # Command to execute
    7879    $command .= " -fileset_id $fileset_id";
    79     $command .= " -file" . join(' -file', @files);
     80    $command .= " -file " . join(' -file ', @files);
    8081    $command .= " -dbname $dbname" if defined $dbname;
    8182

  • trunk/ippScripts/scripts/receive_source.pl

    r23892 r23894  
    5555$uri .= "/index.txt" unless $uri =~ m|/index.txt$|;
    5656my $command = "dsproductls --uri $uri"; # Command to execute
    57 $command .= "--last_fileset $last_fileset" if defined $last_fileset;
     57$command .= " --last_fileset $last_fileset" if defined $last_fileset;
    5858
    5959my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    6161die "Unable to get source listing from $source for $product\n" unless $success;
    6262
    63 # Add filesets
    64 my $last;                   # Last fileset seen
    65 foreach my $line (@$stdout_buf) {
     63# Parse list of filesets
     64my @filesets = ();              # Filesets to add
     65my @lines = split(/\n/, join "", @$stdout_buf); # Lines from output
     66foreach my $line ( @lines ) {
    6667    $line =~ s/\#.*//g;
    6768    next unless $line =~ /\S+/;
    … …  
    6970    my @fields = split(/\s+/, $line); # Fields in line
    7071    my $fileset = $fields[1];
    71     next if not defined $fileset;
    72 
    73     my $command = "receivetool -addfileset"; # Command to execute
    74     $command .= " -source_id $source_id";
    75     $command .= " -fileset $fileset";
    76     $command .= " -dbname $dbname" if defined $dbname;
    77 
    78     unless (defined $no_update) {
    79         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    80             run(command => $command, verbose => $verbose);
    81         die "Unable to add fileset $fileset\n" unless $success;
    82         $last = $fileset;
    83     }
     72    push @filesets, $fileset if defined $fileset and $fileset =~ /\S+/;
    8473}
    8574
    86 # Update the last fileset seen
    87 if (defined $last and (not defined $last_fileset or $last ne $last_fileset)) {
    88     my $command = "receivetool -updatelast"; # Command to execute
    89     $command .= " -source_id $source_id";
    90     $command .= " -fileset $last";
    91     $command .= " -dbname $dbname" if defined $dbname;
     75if (scalar @filesets > 0) {
     76    # Add filesets
     77    {
     78        my $command = "receivetool -addfileset"; # Command to execute
     79        $command .= " -src_id $source_id";
     80        $command .= " -fileset " . join(' -fileset ', @filesets);
     81        $command .= " -dbname $dbname" if defined $dbname;
    9282
    93     unless (defined $no_update) {
    94         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    95             run(command => $command, verbose => $verbose);
    96         die "Unable to update last fileset to $last\n" unless $success;
     83        unless (defined $no_update) {
     84            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     85                run(command => $command, verbose => $verbose);
     86            die "Unable to add filesets from $source for $product\n" unless $success;
     87        }
     88    }
     89
     90    # Update the last fileset seen
     91    {
     92        my $command = "receivetool -updatelast"; # Command to execute
     93        $command .= " -src_id $source_id";
     94        my $last = pop @filesets; # Last fileset
     95        $command .= " -fileset $last";
     96        $command .= " -dbname $dbname" if defined $dbname;
     97
     98        unless (defined $no_update) {
     99            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     100                run(command => $command, verbose => $verbose);
     101            die "Unable to update last fileset to $last\n" unless $success;
     102        }
    97103    }
    98104}

  • trunk/ippTools/share/pxadmin_drop_tables.sql

    r23688 r23894  
    6969DROP TABLE IF EXISTS rcDSFileset;
    7070DROP TABLE IF EXISTS rcRun;
     71DROP TABLE IF EXISTS receiveSource;
     72DROP TABLE IF EXISTS receiveFileset;
     73DROP TABLE IF EXISTS receiveFile;
     74DROP TABLE IF EXISTS receiveResult;
    7175
    7276SET FOREIGN_KEY_CHECKS=1

  • trunk/ippTools/share/receivetool_pendingfile.sql

    r23885 r23894  
    1 SELECT *
     1SELECT
     2    receiveFile.file_id,
     3    source,
     4    product,
     5    workdir,
     6    fileset,
     7    file
    28FROM receiveFile
    39JOIN receiveFileset USING(fileset_id)
    410JOIN receiveSource USING(source_id)
    5 LEFT JOIN receiveResult USING(file_id)
    6 WHERE receiveFile.file_id IS NULL
     11LEFT JOIN receiveResult
     12    ON receiveResult.file_id = receiveFile.file_id
     13WHERE receiveResult.file_id IS NULL

  • trunk/ippTools/src/receivetool.c

    r23885 r23894  
    9595
    9696    // required
    97     PXOPT_LOOKUP_STR(source, config->args, "-stage", true, false);
     97    PXOPT_LOOKUP_STR(source, config->args, "-source", true, false);
    9898    PXOPT_LOOKUP_STR(product, config->args, "-product",  true, false);
     99    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
    99100
    100101    // optional
    101     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
    102102    PXOPT_LOOKUP_STR(comment, config->args, "-comment",  false, false);
    103103    PXOPT_LOOKUP_STR(last, config->args, "-last",  false, false);
    … …  
    168168
    169169    // required
    170     PXOPT_LOOKUP_S64(source_id, config->args, "-source_id", true, false);
    171     PXOPT_LOOKUP_STR(fileset, config->args, "-fileset",  true, false);
    172 
    173     if (!receiveFilesetInsert(config->dbh, 0, source_id, fileset)) {
     170    PXOPT_LOOKUP_S64(source_id, config->args, "-src_id", true, false);
     171    psMetadataItem *filesets = psMetadataLookup(config->args, "-fileset");
     172    if (!filesets) {
     173        psError(PS_ERR_UNKNOWN, true, "-fileset is required");
     174        return false;
     175    }
     176
     177    if (!psDBTransaction(config->dbh)) {
     178        psError(PS_ERR_UNKNOWN, false, "Database error");
     179        return false;
     180    }
     181
     182    psListIterator *iter = psListIteratorAlloc(filesets->data.list, PS_LIST_HEAD, false); // Iterator
     183    psMetadataItem *item = NULL;        // Item from iteration
     184    while ((item = psListGetAndIncrement(iter))) {
     185        psAssert(item && item->data.V && item->type == PS_DATA_STRING, "Argument is bad");
     186
     187        if (!receiveFilesetInsert(config->dbh, 0, source_id, item->data.str)) {
     188            psError(PS_ERR_UNKNOWN, false, "Unable to add fileset");
     189            if (!psDBTransaction(config->dbh)) {
     190                psError(PS_ERR_UNKNOWN, false, "Database error");
     191                return false;
     192            }
     193            psFree(iter);
     194            return false;
     195        }
     196    }
     197    psFree(iter);
     198
     199    if (!psDBCommit(config->dbh)) {
    174200        psError(PS_ERR_UNKNOWN, false, "Database error");
    175201        return false;
    … …  
    184210
    185211    // required
    186     PXOPT_LOOKUP_S64(source_id, config->args, "-source_id", true, false);
     212    PXOPT_LOOKUP_S64(source_id, config->args, "-src_id", true, false);
    187213    PXOPT_LOOKUP_STR(fileset, config->args, "-fileset",  true, false);
    188214
    … …  
    212238    PXOPT_COPY_STR(config->args, where, "-product", "receiveSource.product", "==");
    213239    PXOPT_COPY_STR(config->args, where, "-comment", "receiveSource.comment", "LIKE");
     240
     241    // optional
     242    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     243    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    214244
    215245    psString query = pxDataGet("receivetool_pendingfileset.sql");
    … …  
    227257    psFree(where);
    228258
     259    if (limit) {
     260        psString limitString = psDBGenerateLimitSQL(limit);
     261        psStringAppend(&query, " %s", limitString);
     262        psFree(limitString);
     263    }
     264
    229265    if (!p_psDBRunQueryF(config->dbh, query)) {
    230266        psError(PS_ERR_UNKNOWN, false, "Database error");
    … …  
    244280        return true;
    245281    }
    246     if (!ippdbPrintMetadatas(stdout, output, "receiveFileset", true)) {
     282    if (!ippdbPrintMetadatas(stdout, output, "receiveFileset", !simple)) {
    247283        psError(PS_ERR_UNKNOWN, false, "Failed to print array");
    248284        psFree(output);
    … …  
    260296    // required
    261297    PXOPT_LOOKUP_S64(fileset_id, config->args, "-fileset_id", true, false);
    262     PXOPT_LOOKUP_STR(file, config->args, "-file",  true, false);
    263 
    264     if (!receiveFileInsert(config->dbh, 0, fileset_id, file)) {
     298    psMetadataItem *files = psMetadataLookup(config->args, "-file");
     299    if (!files) {
     300        psError(PS_ERR_UNKNOWN, true, "-file is required");
     301        return false;
     302    }
     303
     304    if (!psDBTransaction(config->dbh)) {
     305        psError(PS_ERR_UNKNOWN, false, "Database error");
     306        return false;
     307    }
     308
     309    psListIterator *iter = psListIteratorAlloc(files->data.list, PS_LIST_HEAD, false); // Iterator
     310    psMetadataItem *item = NULL;        // Item from iteration
     311    while ((item = psListGetAndIncrement(iter))) {
     312        psAssert(item && item->data.V && item->type == PS_DATA_STRING, "Argument is bad");
     313
     314        if (!receiveFileInsert(config->dbh, 0, fileset_id, item->data.str)) {
     315            psError(PS_ERR_UNKNOWN, false, "Unable to add file");
     316            if (!psDBTransaction(config->dbh)) {
     317                psError(PS_ERR_UNKNOWN, false, "Database error");
     318                return false;
     319            }
     320            psFree(iter);
     321            return false;
     322        }
     323    }
     324    psFree(iter);
     325
     326    if (!psDBCommit(config->dbh)) {
    265327        psError(PS_ERR_UNKNOWN, false, "Database error");
    266328        return false;
    … …  
    282344    PXOPT_COPY_S64(config->args, where, "-fileset_id", "receiveFile.file_id", "==");
    283345
     346    // optional
     347    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     348    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     349
    284350    psString query = pxDataGet("receivetool_pendingfile.sql");
    285351    if (!query) {
    … …  
    296362    psFree(where);
    297363
     364    if (limit) {
     365        psString limitString = psDBGenerateLimitSQL(limit);
     366        psStringAppend(&query, " %s", limitString);
     367        psFree(limitString);
     368    }
     369
    298370    if (!p_psDBRunQueryF(config->dbh, query)) {
    299371        psError(PS_ERR_UNKNOWN, false, "Database error");
    … …  
    313385        return true;
    314386    }
    315     if (!ippdbPrintMetadatas(stdout, output, "receiveFile", true)) {
     387    if (!ippdbPrintMetadatas(stdout, output, "receiveFile", !simple)) {
    316388        psError(PS_ERR_UNKNOWN, false, "Failed to print array");
    317389        psFree(output);
    … …  
    328400
    329401    // required
    330     PXOPT_LOOKUP_S64(fileset_id, config->args, "-source_id", true, false);
     402    PXOPT_LOOKUP_S64(file_id, config->args, "-file_id", true, false);
    331403
    332404    // optional
    … …  
    335407    PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false);
    336408
    337     if (!receiveResultInsert(config->dbh, fileset_id, dtime_copy, dtime_extract, fault)) {
     409    if (!receiveResultInsert(config->dbh, file_id, dtime_copy, dtime_extract, fault)) {
    338410        psError(PS_ERR_UNKNOWN, false, "Database error");
    339411        return false;

  • trunk/ippTools/src/receivetoolConfig.c

    r23885 r23894  
    4747    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-source", 0, "define source (required)", NULL);
    4848    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-product", 0, "define product (required)", NULL);
    49     psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-workdir",   0, "define workdir", NULL);
     49    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-workdir",   0, "define workdir (required)", NULL);
    5050    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL);
    5151    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-last", 0, "define last fileset", NULL);
    … …  
    5959    // -addfileset
    6060    psMetadata *addfilesetArgs = psMetadataAlloc();
    61     psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-source_id", 0, "define source_id (required)", 0);
    62     psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-fileset", 0, "define fileset (required)", NULL);
     61    psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-src_id", 0, "define source_id (required)", 0);
     62    psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-fileset", PS_META_DUPLICATE_OK, "define fileset (multiple OK, required)", NULL);
    6363
    6464    // -updatelast
    6565    psMetadata *updatelastArgs = psMetadataAlloc();
    66     psMetadataAddS64(updatelastArgs, PS_LIST_TAIL, "-source_id", 0, "define source_id (required)", 0);
     66    psMetadataAddS64(updatelastArgs, PS_LIST_TAIL, "-src_id", 0, "define source_id (required)", 0);
    6767    psMetadataAddStr(updatelastArgs, PS_LIST_TAIL, "-fileset", 0, "define last fileset (required)", NULL);
    6868
    … …  
    7272    psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-product", 0, "search on product", NULL);
    7373    psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL);
     74    psMetadataAddU64(pendingfilesetArgs, PS_LIST_TAIL, "-limit",  0, "limit result set", 0);
     75    psMetadataAddBool(pendingfilesetArgs, PS_LIST_TAIL, "-simple",  0, "use simple output format?", false);
    7476
    7577    // -addfile
    … …  
    8486    psMetadataAddStr(pendingfileArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL);
    8587    psMetadataAddS64(pendingfileArgs, PS_LIST_TAIL, "-fileset_id", 0, "search on fileset_id", 0);
     88    psMetadataAddU64(pendingfileArgs, PS_LIST_TAIL, "-limit",  0, "limit result set", 0);
     89    psMetadataAddBool(pendingfileArgs, PS_LIST_TAIL, "-simple",  0, "use simple output format?", false);
    8690
    8791    // -addresult
Note: See TracChangeset for help on using the changeset viewer.