Changeset 16867 for trunk/DataStoreServer/scripts/dsreg

Timestamp:

Mar 6, 2008, 5:55:53 PM (18 years ago)

Author:

bills

Message:

dded support for product specific data and file type specific data.
Allow fileset contents to reside outside of the data store directory
and be referenced by symbolic link.
Add option to copy files from a source into the destination fileset.
Removed obsolete variables from the original mock up.

File:

• trunk/DataStoreServer/scripts/dsreg (modified) (14 diffs)

trunk/DataStoreServer/scripts/dsreg

@@ -1 +1 @@
 #!/usr/bin/env perl
 #
-# DataStore fileset registration
-#
-# Does not enforce validity of metadata items.
+# Data Store file set registration and de-registration
 #
 
@@ -29 +27 @@
                        caturi
                        );
+my $product;
+my $fileset;
+
+my $fstype;
+my $ps0;
+my $ps1;
+my $ps2;
+my $ps3;
+my $ps4;
+my $ps5;
+my $ps6;
+my $ps7;
+
+my $linkfiles;      # if set, put links to files in datapath in the Data Store
+my $copyfiles;      # if set, copy files from this directory to the Data Store fileset
+my $datapath;       # source for files required if $linkfiles or $copyfiles
+
+my $ds_dir;
 
 my $add;
 my $del;
-
-my $ds_dir;
-my $product;
-my $fileset;
-
-### XXX: these aren't currently used
-### fill in some bogus values here
-my $type = 'OBJECT';
-my $ra = '00:00:00.00';
-my $dec = '00:00:00.00';
-my $equinox = 2000;
-my $etime = 30.0;
-my $filter = 'z';
-my $airmass = 1.2;
-
-my $date;
+my $remove;
+
+my $verbosity = 1;
+
+
+
 
 #
@@ -53 +59 @@
 #
 
-# XXX: need to make argument parsing file type specific
 GetOptions(
-    'add'               => \$add,
-    'del'               => \$del,
-    'dsdir|ds=s'        => \$ds_dir,
-    'type|t=s'          => \$type,
-    'ra|r=s'            => \$ra,
-    'dec|d=s'           => \$dec,
-    'equinox|eq=f'      => \$equinox,
-    'etime|ex|e=f'      => \$etime,
-    'filter|f=s'        => \$filter,
-        'date=s'        => \$date,
-    'airmass|a=f'       => \$airmass,
+        'add'           =>      \$add,
+        'del'           =>      \$del,
+        'product=s'     =>      \$product,
+        'fileset=s'     =>      \$fileset,
+        'type=s'        =>      \$fstype,
+        'datapath=s'    =>      \$datapath,
+        'link'          =>      \$linkfiles,
+        'copy'          =>      \$copyfiles,
+        'rm'            =>      \$remove,
+        'ps0=s'         =>      \$ps0,          # product specific columns 0 - 7
+        'ps1=s'         =>      \$ps1,
+        'ps2=s'         =>      \$ps2,
+        'ps3=s'         =>      \$ps3,
+        'ps4=s'         =>      \$ps4,
+        'ps5=s'         =>      \$ps5,
+        'ps6=s'         =>      \$ps6,
+        'ps7=s'         =>      \$ps7
 ) or pod2usage(2);
 
 my $err = "";
 
-$err .= "Must specify a product and a fileset.\n"
-    unless @ARGV == 2;
+$err .= "product and fileset are required\n" unless defined $product and $fileset;
 
 # will exit if neither or both -add and -del were specified
 $err .= "Must specify either --add or --del.\n"
     unless $add xor $del;
+
+if ($add) {
+    if (!$fstype) {
+        $err .= "need to specify fileset type to add\n";
+    }
+
+    if ($linkfiles and $copyfiles) {
+        $err .= "only one of --link and --copy allowedn";
+    }
+    if (($linkfiles or $copyfiles) and !$datapath) {
+        $err .= "need to specify datapath with --link or --copy\n";
+    }
+}
+
+
+#
+# lookup the location of the Data Store
+#
 
 my $ipprc =  PS::IPP::Config->new(); # IPP Configuration
@@ -82 +110 @@
     $ds_dir = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_DATA_STORE_ROOT');
     if (!$ds_dir) {
-        die("data store root not defined");
+        die("Data Store root directory not set");
     }
 }
@@ -88 +116 @@
 
 if (!stat("$ds_dir/index.txt")) {
-    $err .= "Datastore not found at '$ds_dir'.\n"
+    $err .= "Data Store not found at '$ds_dir'.\n"
 }
 
@@ -95 +123 @@
     if ($err);
 
-$product = shift;
-
 show_usage("Invalid product '$product'.")
     unless defined stat("$ds_dir/$product/index.txt");
-
-$fileset = shift;
-
-
-#
-# run
-#
 
 my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER');
@@ -117 +136 @@
 my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n";
 
-my $index_script_name = "$ds_dir/$product/$fileset/index.txt";
+
+
+my $fileset_dir = "$ds_dir/$product/$fileset";
+my $index_script_name = "$fileset_dir/index.txt";
 
 if ($del) {
@@ -143 +165 @@
     }
 
-    # zap the index script
-    unlink("$index_script_name");
+    if ($remove) {
+        if (system "rm -r $fileset_dir") {
+            die("failed to remove $fileset_dir");
+        }
+    } else  {
+        # zap the index script
+        unlink("$index_script_name");
+    }
 
     my $fileset_id = $fs_row->{fileset_id};
@@ -170 +198 @@
 } else {
     #
-    # adding a new fileset
+    # add a new fileset
     #
+
+    # make a string out of the product specifc column values provided
+    my $prodcolstr = make_prodcol_str($ps0, $ps1, $ps2, $ps3, $ps4, $ps5, $ps6, $ps7);
+
     my $stmt = $dbh->prepare("SELECT prod_id FROM dsProduct WHERE prod_name = \'$product\'");
     $stmt->execute();
@@ -198 +230 @@
         my $filename;
         my $filetype;
-        my $type_col_0;
-        if (/([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)/) {
-            $filename = $1;
-            $filetype = $2;
-            $type_col_0 = $3;
-        } else {
-            unless (/([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)\s+/) {
+        my $ts_string = "";
+
+        chomp;
+        my @fields = split /\|/;
+        if (@fields < 2) {
                 print STDERR "Line $lineno: ignored malformed input line.\n";
                 next;
-            }
-            $filename = $1;
-            $filetype = $2;
-        }
-
-        my @finfo = stat("$ds_dir/$product/$fileset/$filename");
+        }
+        $filename = shift @fields;
+        $filetype = shift @fields;
+
+        # 
+        # save any type specific fields provided in the string to be added to the
+        # VALUES list for this file
+        my $nfields = @fields;
+        my $i;
+        for ($i=0; $i < $nfields; $i++) {
+            $ts_string .= ", \'$fields[$i]\'";
+        }
+        for ($i=$nfields ; $i < 4; $i++) {
+            $ts_string .= ", NULL";
+        }
+
+        my $path = "$datapath/$filename";
+        my @finfo = stat("$path");
 
         unless (@finfo) {
             die ("Line $lineno: referenced file "
-                ."($ds_dir/$product/$fileset/$1) does not exist.\n");
+                ."($path does not exist.\n");
         }
         
         # Get MD5 sum
-        my $md5 = file_md5_hex("$ds_dir/$product/$fileset/$filename");
+        my $md5 = file_md5_hex($path);
 
         my $hashref = { 
-            'file' => $filename,
-            'bytes' => $finfo[7],
+            'file'      => $filename,
+            'bytes'     => $finfo[7],
             'md5sum'    => $md5,
             'type'      => $filetype,
-            'type_col_0'=> $type_col_0
+            'ts_string'  => $ts_string
         };
 
         push @files, $hashref;
     }
-
 
     if (@files == 0) {
         die("empty filelist");
     }
-
-    # create the cgi script index file by making a copy of its parent's
+    
+    #
+    # Prepare the fileset directory
+    if ($linkfiles or $copyfiles) {
+            ## create the fileset directory
+        if (! -e $fileset_dir) {
+            if (!mkdir $fileset_dir) {
+                die("failed trying to make fileset directory $fileset_dir");
+            }
+        }
+        eval {
+            ## then copy or symlink the files into place
+            foreach my $ref (@files) {
+                my $src = "$datapath/$ref->{file}";
+                my $dest = "$fileset_dir/$ref->{file}";
+
+                if ($linkfiles) {
+                    if (!  symlink $src, $dest) {
+                        die("failed trying to link $src to $dest");
+                    }
+                } else {
+                    if (!copy($src, $dest)) {
+                        die("copy($src, $dest) failed");
+                    }
+                }
+            }
+        };
+        if ($@) { # an error occured
+            print STDERR "error preparing the fileset directory: $@\n";
+
+            if (system "rm -r $fileset_dir") {
+                die("failed to remove $fileset_dir");
+            }
+            
+            exit $PS_EXIT_UNKNOWN_ERROR;
+        }
+    }
+
+    # create the cgi script index.txt by making a copy of its parent's
     if (!copy("$ds_dir/$product/index.txt", $index_script_name)) {
-        die("failed to copy index script to fileset");
+        die("failed to copy index script to file set");
     }
 
@@ -248 +326 @@
         $dbh->{AutoCommit} = 0;
 
-        $count = $dbh->do("INSERT into dsFileset" .
-                " (prod_id, fileset_name, reg_time, type)" .
-                " VALUES($prod_id, \'$fileset\', UTC_TIMESTAMP(), 'PSRESULTS')");
+        my $qstring = "INSERT into dsFileset" .
+                " (prod_id, fileset_name, reg_time, type," .
+                " prod_col_0, prod_col_1, prod_col_2, prod_col_3, prod_col_4, prod_col_5, " .
+                " prod_col_6, prod_col_7)" .
+                # note prodcolstr begins with a comma
+                " VALUES($prod_id, \'$fileset\', UTC_TIMESTAMP(), \'$fstype\' $prodcolstr)";
+        $count = $dbh->do($qstring);
         if ($count == 0E0) {
             die("failed to insert $fileset");
@@ -260 +342 @@
 
         foreach my $ref (@files) {
-            my $query;
-            if ($ref->{type_col_0}) {
-                $query = "INSERT INTO dsFile" .
-                    " (fileset_id, file_id, file_name, bytes, md5sum, type, type_col_0)" .
+            # note: ts_string has a leading comma
+            my $query = "INSERT INTO dsFile" .
+
+                    " (fileset_id, file_id, file_name, bytes, md5sum, type, " .
+                    " type_col_0, type_col_1, type_col_2, type_col_3)" .
+
                     " VALUES($fileset_id, 0, \'$ref->{file}\', $ref->{bytes}," .
-                             " \'$ref->{md5sum}\', \'$ref->{type}\', \'$ref->{type_col_0}\')";
-            } else {
-                $query = "INSERT INTO dsFile" .
-                    " (fileset_id, file_id, file_name, bytes, md5sum, type)" .
-                    " VALUES($fileset_id, 0, \'$ref->{file}\', $ref->{bytes}," .
-                             " \'$ref->{md5sum}\', \'$ref->{type}\')";
-            }
+                             " \'$ref->{md5sum}\', \'$ref->{type}\' $ref->{ts_string} )";
+
             $count = $dbh->do($query);
 
@@ -301 +380 @@
 }
 
+
 sub show_usage {
     my $str = shift;
@@ -310 +390 @@
     pod2usage(
         -msg => 
-"usage: $tmp[$#tmp] [--add|--del] product fileset [options]
+"usage: $tmp[$#tmp] [--add|--del] --product prod_name --fileset fs_name --type fs_type [options]
 
 Commands:
 
-    --del        Remove a fileset.
     --add        Add a new fileset with the given metadata options.
                 File information will be read per-line from STDIN, in the form:
-                    fileID type [chipname]
+
+                    fileID|type|ts0|ts1|ts2|ts3
+
                 These entries are read until EOF is sent.
+                (The type specific ts[0-3]fields are optional)
+
+    --del        Remove a fileset.
+
 Options:
-    
-    --dsdir      Specify the path to the Datastore
+    --link      Link files from datapath to Data Store
+    --copy      Copy files from datapath to Data Store
+    --datapath  path to source files
+    --ps0 - ps7 Optional product specific data
+    --rm        with --del remove the fileset directory from the Data Store
     
 $str",
         -exitval => 2
     );
-
-    if (0) {
-     my $saveString = "(The following metadata is not used yet)
-
-    --date       ISO8601 date string. (File timestamp used if not specified.)
-    --type       Fileset type.
-    --ra         Telescope pointing RA.
-    --dec        Telescope pointing Dec.
-    --equinox    Equinox.
-    --etime      Exposure time.
-    --filter     Filter used.
-    --airmass    Air mass. ";   
-    }
-}
-
-# get the current UTC time in iso8601 format
-# this is used if no time parameter is specified
-sub get_iso8601() {
-    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
-        gmtime(time);
-
-    return sprintf("%4d-%02d-%02dT%02d:%02d:%02dZ",
-        $year+1900, $mon+1, $mday, $hour, $min, $sec);
-}
+}
+
+sub make_prodcol_str {
+
+    my @list = @_;
+    my $string = "";
+
+    foreach my $s (@list) {
+        if ($s) {
+            $string .= ", \'$s\'";
+        } else {
+            $string .= ", NULL";
+        }
+    }
+
+    return $string;
+}