Changeset 29612

Timestamp:

Oct 29, 2010, 2:55:09 PM (16 years ago)

Author:

bills

Message:

detect faults in warps for diff stage components, and destreak failures

File:

• trunk/pstamp/scripts/pstamp_checkdependent.pl (modified) (3 diffs)

trunk/pstamp/scripts/pstamp_checkdependent.pl

@@ -304 +304 @@
 
     my $warp_id = $metadata->{warp_id};
+    my $skycell_id = $metadata->{skycell_id};
     my $state = $metadata->{state};
-    if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop')) {
-        print STDERR "warpRun $warp_id has state $state faulting jobs\n";
+    my $data_state = $metadata->{data_state};
+    if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
+         ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/)) {
+        print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs\n";
         faultJobs('stop', undef, undef, $PSTAMP_GONE);
         exit 0;
+    }
+    if (($state eq 'update') and ($metadata->{fault})) {
+        # fault dependent.
+        my $fault = $metadata->{fault};
+        my_die("warp $warp_id $skycell_id faulted: $fault", $fault);
     }
 
@@ -316 +324 @@
 
         my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id";
-        my $data = runToolAndParse($command, $verbose);
+        my $data = runToolAndParseExpectOne($command, $verbose);
         if (!$data or scalar @$data == 0) {
             # This happens if the chipProcessedImfile disappears which happened when earlier
@@ -671 +679 @@
         } elsif (($dsRun_state eq 'cleaned') or ($dsRun_state eq 'update')) {
             foreach my $c (@$components) {
-                my $command = "$magicdstool -setfiletoupdate -magic_ds_id $magic_ds_id -component $c";
-                $command .= " -set_label $rlabel" if $rlabel;
-                if (!$no_update) {
-                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                        run(command => $command, verbose => $verbose);
-                    unless ($success) {
-                        my_die("failed to set destreaked component to 'update' for ${stage}Run $stage_id $c",
-                            $PS_EXIT_UNKNOWN_ERROR);
+                my $command = "$magicdstool -destreakedfile -magic_ds_id $magic_ds_id -component $c";
+                my $dsfile = runToolAndParseExpectOne($command, $verbose);
+                if (!$dsfile) {
+                    my_die("failed to find magicDSFile for ${stage}Run $stage_id $c", $PS_EXIT_UNKNOWN_ERROR);
+                }
+                if ($dsfile->{fault} > 0) {
+                    my_die("faulted magicDSFile for ${stage}Run $stage_id $c fault: $dsfile->{fault}",
+                        $PS_EXIT_UNKNOWN_ERROR);
+                }
+                if ($dsfile->{data_state} eq 'cleaned') {
+                    $command = "$magicdstool -setfiletoupdate -magic_ds_id $magic_ds_id -component $c";
+                    $command .= " -set_label $rlabel" if $rlabel;
+                    if (!$no_update) {
+                        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                            run(command => $command, verbose => $verbose);
+                        unless ($success) {
+                            my_die("failed to set destreaked component to 'update' for ${stage}Run $stage_id $c",
+                                $PS_EXIT_UNKNOWN_ERROR);
+                        }
+                    } else {
+                        print "skipping $command\n";
                     }
                 } else {
-                    print "skipping $command\n";
+                    print "magicDSFile for ${stage}Run $stage_id $c data_state: $dsfile->{data_state}\n";
                 }
             }