Index: trunk/DataStoreServer/scripts/dsreg =================================================================== --- trunk/DataStoreServer/scripts/dsreg (revision 16690) +++ trunk/DataStoreServer/scripts/dsreg (revision 16761) @@ -1,5 +1,5 @@ #!/usr/bin/env perl # -# Mock-up datastore registration +# DataStore fileset registration # # Does not enforce validity of metadata items. @@ -7,16 +7,35 @@ use strict; +use warnings; use Getopt::Long qw( GetOptions ); use Pod::Usage qw( pod2usage ); use Digest::MD5::File qw( file_md5_hex ); +use DBI; +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + metadataLookupStr + metadataLookupBool + caturi + ); my $add; my $del; -my $ds_dir = './dsroot'; +#my $ds_dir = './dsroot'; +my $ds_dir = '/var/www/html/ds/dsroot'; my $product; my $fileset; +### XXX: these aren't currently used ### fill in some bogus values here my $type = 'OBJECT'; @@ -34,4 +53,5 @@ # +# XXX: need to make argument parsing file type specific GetOptions( 'add' => \$add, @@ -44,19 +64,19 @@ 'etime|ex|e=f' => \$etime, 'filter|f=s' => \$filter, - 'date=s' => \$date, + 'date=s' => \$date, 'airmass|a=f' => \$airmass, ) or pod2usage(2); -my $err; +my $err = ""; $err .= "Must specify a product and a fileset.\n" - unless $#ARGV == 1; + unless @ARGV == 2; # will exit if neither or both -add and -del were specified -$err .= "Must specify either -add or -del.\n" +$err .= "Must specify either --add or --del.\n" unless $add xor $del; $err .= "Datastore not found at '$ds_dir'.\n" - if stat("$ds_dir/index.txt") == undef; + if stat("$ds_dir/index.txt") eq undef; # bail if any of the above args were improper @@ -67,5 +87,5 @@ show_usage("Invalid product '$product'.") - unless stat("$ds_dir/$product/index.txt") != undef; + unless defined stat("$ds_dir/$product/index.txt"); $fileset = shift; @@ -78,4 +98,17 @@ umask 0011; +my $ipprc = PS::IPP::Config->new(); # IPP Configuration +my $siteConfig = $ipprc->{_siteConfig}; + +my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER'); +my $dbname = metadataLookupStr($siteConfig, 'DBNAME'); +my $dbuser = metadataLookupStr($siteConfig, 'DBUSER'); +my $dbpass = metadataLookupStr($siteConfig, 'DBPASSWORD'); +exit ($PS_EXIT_CONFIG_ERROR) unless defined $dbserver and $dbname and $dbuser and $dbpass; + +my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n"; + # @@ -83,4 +116,7 @@ # if ($del) { + + die("delete not implemented yet"); + if (stat("$ds_dir/$product/$fileset/index.txt") == undef) { print "Fileset '$fileset' doesn't exist under $product.\n"; @@ -178,22 +214,22 @@ # -# add fileset +# addding a new fileset # else { - unless (stat("$ds_dir/$product/$fileset/index.txt") == undef) { + my $stmt = $dbh->prepare("SELECT prod_id FROM dsProduct WHERE prod_name = \'$product\'"); + $stmt->execute(); + my $row = $stmt->fetchrow_hashref(); + my $prod_id = $row->{prod_id}; + + if (!$prod_id) { + die("product $product not found"); + } + my $count = $dbh->do("SELECT fileset_id FROM dsFileset WHERE fileset_name = '$fileset'" . + " AND prod_id = $prod_id"); + + if ($count != 0E0) { print "Fileset '$fileset' already exists under $product.\n"; - exit(4); - } - - - # Create the fileset index - - open(IO, ">", "$ds_dir/$product/$fileset/index.txt") - or die("Failed to open '$ds_dir/$product/$fileset/index.txt' for writing"); - print IO "# fileID |bytes |md5sum |type|chipname\n"; - - #print "-- Send file list with lines of the form 'fileID type chipname' " - # ."followed by EOF.\n"; - + exit 1; + } # Read file data from STDIN @@ -201,82 +237,105 @@ my $lineno = 0; + my @files; while () { $lineno++; - unless (/([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)/) { - print "Line $lineno: ignored malformed input line.\n"; - next; - } - - # $1 = file id - # $2 = type - # $3 = chip - - my @finfo = stat("$ds_dir/$product/$fileset/$1"); + my $filename; + my $filetype; + my $type_col_0; + if (/([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)/) { + $filename = $1; + $filetype = $2; + $type_col_0 = $3; + } else { + unless (/([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)\s+/) { + print STDERR "Line $lineno: ignored malformed input line.\n"; + next; + } + $filename = $1; + $filetype = $2; + } + + my @finfo = stat("$ds_dir/$product/$fileset/$filename"); unless (@finfo) { - print "Line $lineno: referenced file " - ."($ds_dir/$product/$fileset/$1) does not exist.\n"; - next; + die ("Line $lineno: referenced file " + ."($ds_dir/$product/$fileset/$1) does not exist.\n"); } # Get MD5 sum - my $md5 = file_md5_hex("$ds_dir/$product/$fileset/$1"); - - # Write line to the fileset index - print IO "$1|$finfo[7]|$md5|$2|$3\n"; - } - - close(IO); - + my $md5 = file_md5_hex("$ds_dir/$product/$fileset/$filename"); + + my $hashref = { + 'file' => $filename, + 'bytes' => $finfo[7], + 'md5sum' => $md5, + 'type' => $filetype, + 'type_col_0'=> $type_col_0 + }; + + push @files, $hashref; + } + + + if (@files == 0) { + die("empty filelist"); + } + + eval { + $dbh->{RaiseError} = 1; # raise exception if error occurs + $dbh->{PrintError} = 0; + $dbh->{AutoCommit} = 0; + + $count = $dbh->do("INSERT into dsFileset" . + " (prod_id, fileset_name, reg_time, type)" . + " VALUES($prod_id, \'$fileset\', UTC_TIMESTAMP(), 'PSRESULTS')"); + if ($count == 0E0) { + die("failed to insert $fileset"); + } + + my $fileset_id = $dbh->last_insert_id(undef, undef, undef, undef); + + # print STDERR "Inserted fileset_id $fileset_id\n"; + + foreach my $ref (@files) { + my $query; + if ($ref->{type_col_0}) { + $query = "INSERT INTO dsFile" . + " (fileset_id, file_id, file_name, bytes, md5sum, type, type_col_0)" . + " VALUES($fileset_id, 0, \'$ref->{file}\', $ref->{bytes}," . + " \'$ref->{md5sum}\', \'$ref->{type}\', \'$ref->{type_col_0}\')"; + } else { + $query = "INSERT INTO dsFile" . + " (fileset_id, file_id, file_name, bytes, md5sum, type)" . + " VALUES($fileset_id, 0, \'$ref->{file}\', $ref->{bytes}," . + " \'$ref->{md5sum}\', \'$ref->{type}\')"; + } + $count = $dbh->do($query); + + if ($count == 0E0) { + $dbh->rollback(); + die("failed to insert $ref->{file} into $fileset"); + } + } + + $count = $dbh->do("UPDATE dsProduct SET last_update = UTC_TIMESTAMP(), last_fs = \'$fileset\'" + . " WHERE prod_id = $prod_id"); + if ($count == 0E0) { + $dbh->rollback(); + die("failed to update dsProduct $prod_id"); + } + + $dbh->commit(); + }; + if ($@) { # an error occured + print STDERR "transactionfailed, rolling back error was:\n$@\n"; + eval {$dbh->rollback();}; + exit 1; + } - # Add reference to the new fileset to the product index - - $date = get_iso8601() if not defined($date); - - #FIXME this should probably be inserted in order! - - my $indexstr = - "$fileset|$date|$type|$ra $dec $equinox|$etime|$filter|$airmass"; - - #open(IO, ">>", "$ds_dir/$product/index.txt") - # or die("Failed to open $ds_dir/$product/index.txt\n"); - open(IO, ">>", "$ds_dir/$product/index.lst") - or die("Failed to open $ds_dir/$product/index.lst\n"); - print IO $indexstr, "\n"; - close(IO); - - - # Update the root listing to reflect the new most-recent fileset - - open(IO, "$ds_dir/index.txt") - or die("Failed to open '$ds_dir/index.txt' for reading\n"); - - my $buf; - - while () { - my ($prod, $oldrecent, $oldtime, $extra) = split(/\|/, $_, 4); - $prod =~ s/^\s+|\s+$//; - - if ($prod ne $product) { - $buf .= $_; - next; - } - - $buf .= "$prod|$fileset|$date|$extra"; - } - - close(IO); - - - # Write the new index - - open(IO, ">", "$ds_dir/index.txt") - or die("Failed to open '$ds_dir/index.txt' for writing\n"); - - print IO $buf; - close(IO); - print "Added $fileset to $product.\n"; + + exit 0; } @@ -290,10 +349,10 @@ pod2usage( -msg => -"usage: $tmp[$#tmp] [-add|-del] product fileset [options] +"usage: $tmp[$#tmp] [--add|--del] product fileset [options] Commands: - -del Remove a fileset. - -add Add a new fileset with the given metadata options. + --del Remove a fileset. + --add Add a new fileset with the given metadata options. File information will be read per-line from STDIN, in the form: fileID type chipname @@ -301,16 +360,16 @@ Options: - -dsdir Specify the path to the Datastore (defaults to CWD). + --dsdir Specify the path to the Datastore (defaults to CWD). (The following metadata is optional because this is just a mock-up.) - -date ISO8601 date string. (File timestamp used if not specified.) - -type Fileset type. - -ra Telescope pointing RA. - -dec Telescope pointing Dec. - -equinox Equinox. - -etime Exposure time. - -filter Filter used. - -airmass Air mass. + --date ISO8601 date string. (File timestamp used if not specified.) + --type Fileset type. + --ra Telescope pointing RA. + --dec Telescope pointing Dec. + --equinox Equinox. + --etime Exposure time. + --filter Filter used. + --airmass Air mass. $str",