Changeset 16761 for trunk/DataStoreServer/scripts

Timestamp:

Feb 29, 2008, 2:35:40 PM (18 years ago)

Author:

bills

Message:

converted --add to work with data base based DataStore

File:

• trunk/DataStoreServer/scripts/dsreg (modified) (11 diffs)

trunk/DataStoreServer/scripts/dsreg

@@ -1 +1 @@
 #!/usr/bin/env perl
 #
-# Mock-up datastore registration
+# DataStore fileset registration
 #
 # Does not enforce validity of metadata items.
@@ -7 +7 @@
 
 use strict;
+use warnings;
 
 use Getopt::Long qw( GetOptions );
 use Pod::Usage qw( pod2usage );
 use Digest::MD5::File qw( file_md5_hex ); 
+use DBI;
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::Stats;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config qw($PS_EXIT_SUCCESS
+                       $PS_EXIT_UNKNOWN_ERROR
+                       $PS_EXIT_SYS_ERROR
+                       $PS_EXIT_CONFIG_ERROR
+                       $PS_EXIT_PROG_ERROR
+                       $PS_EXIT_DATA_ERROR
+                       $PS_EXIT_TIMEOUT_ERROR
+                       metadataLookupStr
+                       metadataLookupBool
+                       caturi
+                       );
 
 my $add;
 my $del;
 
-my $ds_dir = './dsroot';
+#my $ds_dir = './dsroot';
+my $ds_dir = '/var/www/html/ds/dsroot';
 my $product;
 my $fileset;
 
+### XXX: these aren't currently used
 ### fill in some bogus values here
 my $type = 'OBJECT';
@@ -34 +53 @@
 #
 
+# XXX: need to make argument parsing file type specific
 GetOptions(
     'add'               => \$add,
@@ -44 +64 @@
     'etime|ex|e=f'      => \$etime,
     'filter|f=s'        => \$filter,
-        'date=s'                        => \$date,
+        'date=s'        => \$date,
     'airmass|a=f'       => \$airmass,
 ) or pod2usage(2);
 
-my $err;
+my $err = "";
 
 $err .= "Must specify a product and a fileset.\n"
-    unless $#ARGV == 1;
+    unless @ARGV == 2;
 
 # will exit if neither or both -add and -del were specified
-$err .= "Must specify either -add or -del.\n"
+$err .= "Must specify either --add or --del.\n"
     unless $add xor $del;
 
 $err .= "Datastore not found at '$ds_dir'.\n"
-    if stat("$ds_dir/index.txt") == undef;
+    if stat("$ds_dir/index.txt") eq undef;
 
 # bail if any of the above args were improper
@@ -67 +87 @@
 
 show_usage("Invalid product '$product'.")
-    unless stat("$ds_dir/$product/index.txt") != undef;
+    unless defined stat("$ds_dir/$product/index.txt");
 
 $fileset = shift;
@@ -78 +98 @@
 umask 0011;
 
+my $ipprc =  PS::IPP::Config->new(); # IPP Configuration
+my $siteConfig = $ipprc->{_siteConfig};
+
+my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER');
+my $dbname   = metadataLookupStr($siteConfig, 'DBNAME');
+my $dbuser   = metadataLookupStr($siteConfig, 'DBUSER');
+my $dbpass   = metadataLookupStr($siteConfig, 'DBPASSWORD');
+exit ($PS_EXIT_CONFIG_ERROR) unless defined $dbserver and $dbname and $dbuser and $dbpass;
+
+my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n";
+
 
 #
@@ -83 +116 @@
 #
 if ($del) {
+
+    die("delete not implemented yet");
+
     if (stat("$ds_dir/$product/$fileset/index.txt") == undef) {
         print "Fileset '$fileset' doesn't exist under $product.\n";
@@ -178 +214 @@
 
 #
-# add fileset
+# addding a new fileset
 #
 else {
-    unless (stat("$ds_dir/$product/$fileset/index.txt") == undef) {
+    my $stmt = $dbh->prepare("SELECT prod_id FROM dsProduct WHERE prod_name = \'$product\'");
+    $stmt->execute();
+    my $row = $stmt->fetchrow_hashref();
+    my $prod_id = $row->{prod_id};
+
+    if (!$prod_id) {
+        die("product $product not found");
+    }
+    my $count = $dbh->do("SELECT fileset_id FROM dsFileset WHERE fileset_name = '$fileset'" .
+                        " AND prod_id = $prod_id");
+
+    if ($count != 0E0) {
         print "Fileset '$fileset' already exists under $product.\n";
-        exit(4);
-    }
-
-    
-        # Create the fileset index
-
-    open(IO, ">", "$ds_dir/$product/$fileset/index.txt")
-        or die("Failed to open '$ds_dir/$product/$fileset/index.txt' for writing");
-    print IO "# fileID      |bytes   |md5sum                          |type|chipname\n";
-    
-    #print "-- Send file list with lines of the form 'fileID type chipname' "
-    #    ."followed by EOF.\n";
-
+        exit 1;
+    }
 
     # Read file data from STDIN
@@ -201 +237 @@
     my $lineno = 0;
 
+    my @files;
     while (<STDIN>) {
         $lineno++;
         
-        unless (/([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)/) {
-            print "Line $lineno: ignored malformed input line.\n";
-            next;
-        }
-
-        # $1 = file id
-        # $2 = type
-        # $3 = chip
-
-        my @finfo = stat("$ds_dir/$product/$fileset/$1");
+        my $filename;
+        my $filetype;
+        my $type_col_0;
+        if (/([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)/) {
+            $filename = $1;
+            $filetype = $2;
+            $type_col_0 = $3;
+        } else {
+            unless (/([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)\s+/) {
+                print STDERR "Line $lineno: ignored malformed input line.\n";
+                next;
+            }
+            $filename = $1;
+            $filetype = $2;
+        }
+
+        my @finfo = stat("$ds_dir/$product/$fileset/$filename");
 
         unless (@finfo) {
-            print "Line $lineno: referenced file "
-                ."($ds_dir/$product/$fileset/$1) does not exist.\n";
-            next;
+            die ("Line $lineno: referenced file "
+                ."($ds_dir/$product/$fileset/$1) does not exist.\n");
         }
         
         # Get MD5 sum
-        my $md5 = file_md5_hex("$ds_dir/$product/$fileset/$1");
-
-        # Write line to the fileset index
-        print IO "$1|$finfo[7]|$md5|$2|$3\n";
-    }
-
-    close(IO);
-    
+        my $md5 = file_md5_hex("$ds_dir/$product/$fileset/$filename");
+
+        my $hashref = { 
+            'file' => $filename,
+            'bytes' => $finfo[7],
+            'md5sum'    => $md5,
+            'type'      => $filetype,
+            'type_col_0'=> $type_col_0
+        };
+
+        push @files, $hashref;
+    }
+
+
+    if (@files == 0) {
+        die("empty filelist");
+    }
+
+    eval {
+        $dbh->{RaiseError} = 1; # raise exception if error occurs
+        $dbh->{PrintError} = 0;
+        $dbh->{AutoCommit} = 0;
+
+        $count = $dbh->do("INSERT into dsFileset" .
+                " (prod_id, fileset_name, reg_time, type)" .
+                " VALUES($prod_id, \'$fileset\', UTC_TIMESTAMP(), 'PSRESULTS')");
+        if ($count == 0E0) {
+            die("failed to insert $fileset");
+        }
+
+        my $fileset_id = $dbh->last_insert_id(undef, undef, undef, undef);
+
+        # print STDERR "Inserted fileset_id $fileset_id\n";
+
+        foreach my $ref (@files) {
+            my $query;
+            if ($ref->{type_col_0}) {
+                $query = "INSERT INTO dsFile" .
+                    " (fileset_id, file_id, file_name, bytes, md5sum, type, type_col_0)" .
+                    " VALUES($fileset_id, 0, \'$ref->{file}\', $ref->{bytes}," .
+                             " \'$ref->{md5sum}\', \'$ref->{type}\', \'$ref->{type_col_0}\')";
+            } else {
+                $query = "INSERT INTO dsFile" .
+                    " (fileset_id, file_id, file_name, bytes, md5sum, type)" .
+                    " VALUES($fileset_id, 0, \'$ref->{file}\', $ref->{bytes}," .
+                             " \'$ref->{md5sum}\', \'$ref->{type}\')";
+            }
+            $count = $dbh->do($query);
+
+            if ($count == 0E0) {
+                $dbh->rollback();
+                die("failed to insert $ref->{file} into $fileset");
+            }
+        }
+
+        $count = $dbh->do("UPDATE dsProduct SET last_update = UTC_TIMESTAMP(), last_fs = \'$fileset\'"
+            . " WHERE prod_id = $prod_id");
+        if ($count == 0E0) {
+            $dbh->rollback();
+            die("failed to update dsProduct $prod_id");
+        }
+
+        $dbh->commit();
+    };
+    if ($@) { # an error occured
+        print STDERR "transactionfailed, rolling back error was:\n$@\n";
+        eval {$dbh->rollback();};
+        exit 1;
+    }
         
-    # Add reference to the new fileset to the product index
-
-        $date = get_iso8601() if not defined($date);
-
-        #FIXME this should probably be inserted in order!
-
-    my $indexstr =
-        "$fileset|$date|$type|$ra  $dec $equinox|$etime|$filter|$airmass";
-
-    #open(IO, ">>", "$ds_dir/$product/index.txt")
-    #    or die("Failed to open $ds_dir/$product/index.txt\n");
-    open(IO, ">>", "$ds_dir/$product/index.lst")
-        or die("Failed to open $ds_dir/$product/index.lst\n");
-    print IO $indexstr, "\n";
-    close(IO);
-
-
-     # Update the root listing to reflect the new most-recent fileset
-
-    open(IO, "$ds_dir/index.txt")
-        or die("Failed to open '$ds_dir/index.txt' for reading\n");
-
-    my $buf;
-
-    while (<IO>) {
-        my ($prod, $oldrecent, $oldtime, $extra) = split(/\|/, $_, 4);
-        $prod =~ s/^\s+|\s+$//;
-        
-        if ($prod ne $product) {
-            $buf .= $_;
-            next;
-        }
-
-        $buf .= "$prod|$fileset|$date|$extra";
-    }
-
-    close(IO);
-
-
-    # Write the new index
-
-    open(IO, ">", "$ds_dir/index.txt")
-        or die("Failed to open '$ds_dir/index.txt' for writing\n");
-        
-    print IO $buf;
-    close(IO);
-    
     print "Added $fileset to $product.\n";
+
+    exit 0;
 }
 
@@ -290 +349 @@
     pod2usage(
         -msg => 
-"usage: $tmp[$#tmp] [-add|-del] product fileset [options]
+"usage: $tmp[$#tmp] [--add|--del] product fileset [options]
 
 Commands:
 
-    -del        Remove a fileset.
-    -add        Add a new fileset with the given metadata options.
+    --del        Remove a fileset.
+    --add        Add a new fileset with the given metadata options.
                 File information will be read per-line from STDIN, in the form:
                     fileID type chipname
@@ -301 +360 @@
 Options:
     
-    -dsdir      Specify the path to the Datastore (defaults to CWD).
+    --dsdir      Specify the path to the Datastore (defaults to CWD).
     
  (The following metadata is optional because this is just a mock-up.)
 
-    -date       ISO8601 date string. (File timestamp used if not specified.)
-    -type       Fileset type.
-    -ra         Telescope pointing RA.
-    -dec        Telescope pointing Dec.
-    -equinox    Equinox.
-    -etime      Exposure time.
-    -filter     Filter used.
-    -airmass    Air mass.
+    --date       ISO8601 date string. (File timestamp used if not specified.)
+    --type       Fileset type.
+    --ra         Telescope pointing RA.
+    --dec        Telescope pointing Dec.
+    --equinox    Equinox.
+    --etime      Exposure time.
+    --filter     Filter used.
+    --airmass    Air mass.
     
 $str",