Changeset 38281

Timestamp:

May 15, 2015, 3:50:54 PM (11 years ago)

Author:

watersc1

Message:

Untested but complete version of the component check/replicate/cull code.

File:

• trunk/tools/stsci_disks/p2_check.pl (modified) (6 diffs)

trunk/tools/stsci_disks/p2_check.pl

@@ -33 +33 @@
 ) || pod2usage(2 );
 
+
+my $server = $ENV{'NEB_SERVER'};
+my $neb = Nebulous::Client->new(
+    proxy => "$server",
+    );
+
 my $camera = 'GPC1';
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set_up", $stage, $stage_id);
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
+my $time = time();
+my $logDest = "neb://any/stsci_shuffle/${stage}/${stage}.${stage_id}.log.${time}";
+my $ipprc = PS::IPP::Config->new( "GPC1" ) or &my_die( "Could not create config object.", $stage,$stage_id);
+$ipprc->redirect_output($logDest) or &my_die("Could not redirect output to logfile ${logDest}", $stage,$stage_id);
+
+my $manifest_file = $ipprc->file_resolve("neb://any/stsci_shuffle/${stage}/${stage}.${stage_id}.MANIFEST",1);
+open(MANIFEST,">${manifest_file}") || &my_die( "Could not open manifest file", $stage,$stage_id);
 
 # Define which things we want to look at, and where and how many of each.
@@ -58 +71 @@
                    'PPSTACK.UNCONV.EXPWT' => { 'local' => 2, 'stsci' => 1 },
                    'PSPHOT.PSF.SKY.SAVE' => { 'local' => 2, 'stsci' => 1 },
-                   'PSPHOT.OUT.CMF.MEF' => { 'local' => 2, 'stsci' => 1 },
+                   'PSPHOT.STACK.OUTPUT' => { 'local' => 2, 'stsci' => 1 },
                    'PPSTACK.CONFIG' => { 'local' => 2, 'stsci' => 1}
 
@@ -85 +98 @@
 my @keylist = ();
 
+
 my $otime = $time;
 $time = time();
@@ -97 +111 @@
     
     foreach my $component (keys %components) {
-        my $key = $ipprc->filename($component,$path_base);
+        my $key;
+        if (($stage eq 'stack')&&($component eq 'PSPHOT.STACK.OUTPUT')) {
+            # Do special things to get to skycalresult
+            my $ss_command = "$staticskytool -skycalresult -stack_id $stage_id -label LAP.PV3.20140730.skycal01 -dbname $dbname";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to run STAGEtool -liststagestuff ", $stage,$stage_id);
+            }
+            my $SS_MDlist = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to run STAGEtool -liststagestuff ", $stage,$stage_id);
+            my $SS_data = parse_md_list($MDlist);
+            my $SSRun = ${ $SS_data }[0];
+            my $SS_pb = $SSRun->{path_base};
+            $key = $ipprc->filename($component,$SS_pb);
+        }
+        else {
+            $key = $ipprc->filename($component,$path_base);
+        }
 
         # Stealing most of neb-stat here
@@ -117 +149 @@
                 $md5sum_uniq{$md5sums[$i]} = 1;
                 if ($volume =~ /stsci/) { 
-                    $stsci_count++; 
-                    $stsci_copies[$i] = 1;
-                    $local_copies[$i] = 0;
+                    push @stsci_copies, $files[$i];
                 }
                 elsif ($volume =~ /ipp/) { 
-                    $local_count++; 
-                    $stsci_copies[$i] = 0;
-                    $local_copies[$i] = 1;
+                    push @local_copies, $files[$i];
                 }
                 else { &my_die("You can't exist somewhere not ipp and not stsci",$stage,$stage_id,$files[$i]); }
-                
-            }
-        }
-        
+            }
+        }
+        
+        # Check that we have only one md5sum value.
+        if (scalar(keys %md5sum_uniq) != 1) {
+            &my_die("Key $key has inconsistent md5sum values", $stage,$stage_id);
+        }
+        $md5sum{$key} = (keys %md5sum_uniq)[0];
+
         # Do the logic at what we found
-
-                
-                
-    
-    
-
-
-
-
-my $server = $ENV{'NEB_SERVER'};
-my $neb = Nebulous::Client->new(
-    proxy => "$server",
-    );
-
-
-foreach my $subdir (@subdirectories) {
-    my @files = <${subdir}/*>;
-    
-    my @belongs;
-    my @keys;
-    my @vols;
-    my @md5sums;
-
-# Pre scan
-    for (my $i = 0; $i <= $#files; $i++) {
-        my $f = $files[$i];
-        my ($ins_id, $belongs);
-        $ins_id = $f;
-        $ins_id =~ s/^$subdir\///;
-        $ins_id =~ s/\..*//;
-
-        $belongs[$i] = 0; 
-
-        if ($ins_id > 4956576912) { 
-            if ($f =~ /LAP.PV3.20140730/) { # Only PV3
-                if (($f =~ /wrp/)) {
-                    if (($f =~ /fits/)||($f =~ /cmf/)||($f =~ /psf/)) {
-                        $belongs[$i] = 1;
-                    }
-                }
-                if (($f =~ /stk/)&&($f =~ /fits/)&&($f =~ /unconv/)) {
-                    $belongs[$i] = 1;
-                }
-            }
-        }
-        
-        if ($belongs[$i] == 0) {
-            $keys[$i] = $f;
-            $keys[$i] =~ s/.*\/${ins_id}\.//;
-            $keys[$i] =~ s%:%/%g;
-            # Compressed fits 
-            if ($keys[$i] =~ /.fits.fz$/) { 
-                $keys[$i] =~ s/.fits.fz$/.fits/;
-            }
-
-            $md5sums[$i] = local_md5sum($f);
-            # Get a target
-            $vols[$i] = get_volume($keys[$i]);
-            if (defined($vols[$i])) {
-                $vols[$i] = $vols[$i] . ".0";
-            }
-        }
-        elsif ($belongs[$i] == 1) {
-            $md5sums[$i] = 'NOT_CALCULATED';
-            $vols[$i]    = 'ippXXX.Y';
-            $keys[$i]    = 'NO_REPLICATE';
-        }
-        if (defined($vols[$i])) {
-            print "## $ins_id $md5sums[$i] $vols[$i] $belongs[$i] $f $keys[$i] \n";
-        }
-        else {
-            print "## $ins_id $md5sums[$i] NOVOL_BECAUSE_UNKNOWN $belongs[$i] $f $keys[$i] \n";
-        }
-    }
+        if ($#stsci_copies + 1 < $components{$component}{stsci}) {
+            # We need to make an stsci copy.
+            my $vol = choose_random_stsci();
+            push @{ $replicates{$key} }, $vol;
+        }
+        if ($#local_copies + 1 < $components{$component}{local}) {
+            my $vol = get_volume($key);
+            while ($vol =~ /stsci/) {
+                $vol = get_volume($key);
+            }
+            push @{ $replicates{$key} }, $vol;
+        }
+        if ($#stsci_copies + 1 > $components{$component}{stsci}) {
+            for (my $i = $components{$component}{stsci} + 1; $i <= $#stsci_copies; $i++) {
+                push @{ $culls{$key} }, $stsci_copies[$i];
+            }
+        }
+        if ($#local_copies + 1 > $components{$component}{stsci}) {
+            for (my $i = $components{$component}{local} + 1; $i <= $#local_copies; $i++) {
+                push @{ $culls{$key} }, $local_copies[$i];
+            }
+        }
+    }
+    log_print("SCAN $stage $stage_id $path_base");
+
+    foreach my $key (keys %replicates) {
+        foreach my $vol (@{ $replicates{$key} }) {
+            $neb->replicate($key,$vol) or &my_die "failed to replicate a copy of $key to $vol";
+            if ($@) { die $@; }
+        }
+    }
+    system("sync") == 0 or die "Couldn't sync?";
+    log_print("REPL $stage $stage_id $path_base");
+
+    foreach my $key (keys %replicates) {
+        foreach my $vol (@{ $replicates{$key} }) {
+            my $uris = $neb->find_instances($key,$vol);
+            @$uris = map {URI->new($_)->file if $_} @$uris;
+            $uri = ${ $uris }[0];
+            my $tmpmd5 = local_md5sum($uri);
+            if ($tmpmd5 ne $md5sums{$key}) {
+                &my_die "Post-replication md5sum does not match for $key on $vol! $tmpmd5 != $md5sums{$key}";
+            }
+        }
+    }
+    log_print("CHECK $stage $stage_id $path_base");
+    foreach my $key (keys %culls) {
+        foreach my $vol (@{ $culls{$key} }) {
+            $neb->cull($key,$vol);
+        }
+    }
+    log_print("CULL $stage $stage_id $path_base");
+}
+    
+    
+
+        
+sub log_print {
+    my $msg = shift;
     $otime = $time;
     $time = time();
     $dtime = $time - $otime;
-    print "LOG SCAN $subdir $time $dtime\n";
-#    die;
-# Do a replication pass
-    for (my $i = 0; $i <= $#files; $i++) {
-        if ($belongs[$i] == 0) {
-            if (defined($vols[$i])) {
-                $neb->replicate($keys[$i],$vols[$i]) or die "failed to replicate a copy to the backup node";
-                if ($@) { die $@; }
-            }
-            else {
-                # Key wasn't defiend
-                my $dir = dirname($keys[$i]);
-                system("mkdir -p ${unknown_dir}/${dir}");
-                system("cp -a $files[$i] ${unknown_dir}/$keys[$i]");
-            }
-            system("sync") == 0 or die "Couldn't sync?";
-        }
-    }
-    $otime = $time;
-    $time = time();
-    $dtime = $time - $otime;
-    print "LOG REPL $subdir $time $dtime\n";
-
-# Do the check pass
-    for (my $i = 0; $i <= $#files; $i++) {
-        my $uri;
-        if ($belongs[$i] == 0) {
-            if (defined($vols[$i])) {
-                my $uris = $neb->find_instances($keys[$i],$vols[$i]);
-                @$uris = map {URI->new($_)->file if $_} @$uris;
-                $uri = ${ $uris }[0];
-            }
-            else {
-                $uri = "${unknown_dir}/$keys[$i]";
-            }
-
-            my $tmpmd5 = local_md5sum($uri);
-            if ($tmpmd5 ne $md5sums[$i]) { 
-                die "Post-replication md5sum does not match! $tmpmd5 != $md5sums[$i]";
-            }
-            # End my best validation thought.       
-
-            if (defined($vols[$i])) {
-                $neb->cull($keys[$i],$host . "." . $volume);
-            }
-            else {
-                system("rm $files[$i]");
-            }           
-        }
-    }
-
-    $otime = $time;
-    $time = time();
-    $dtime = $time - $otime;
-    print "LOG END $subdir $time $dtime\n";
-    die;
+    print "LOG $msg $time $dtime\n";
 }
 
@@ -319 +282 @@
 }
     
-
-sub secure_replicull {
-    my $key = shift;
-    my $md5sum = shift;
-    my $rep_vol = shift;
-    my $del_vol = shift;
-
-    $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
-    if ($@) { die $@; }
-
-    # Begin my best validation thought
-    system("sync") == 0 or die "Couldn't sync?";
-    my $uris = $neb->find_instances($key,$rep_vol);
-    @$uris = map {URI->new($_)->file if $_} @$uris;
-    my $tmpmd5 = local_md5sum(${ $uris }[0]);
-    if ($tmpmd5 ne $md5sum) { 
-        sleep(2);
-        $tmpmd5 = local_md5sum(${ $uris }[0]);
-        if ($tmpmd5 ne $md5sum) { 
-            die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-        }
-    }
-    # End my best validation thought.
-    
-    $neb->cull($key,$del_vol);
-}
 
 sub local_md5sum {