Changeset 27795

Timestamp:

Apr 28, 2010, 2:18:01 PM (16 years ago)

Author:

bills

Message:

first crack at queuing automatic updates for diffSkyfiles. Various bug fixes found in testing

Location:

trunk

Files:

• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (1 diff)
• ippTasks/pstamp.pro (modified) (1 diff)
• ippTools/share/Makefile.am (modified) (2 diffs)
• ippTools/share/pstamptool_getdependent.sql (added)
• ippTools/share/pstamptool_pendingdependent.sql (modified) (1 diff)
• ippTools/share/pstamptool_revertdependent.sql (modified) (1 diff)
• ippTools/src/pstamptool.c (modified) (5 diffs)
• ippTools/src/pstamptoolConfig.c (modified) (1 diff)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (6 diffs)
• pstamp/scripts/pstampparse.pl (modified) (3 diffs)

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -455 +455 @@
     my $inverse = $option_mask & $PSTAMP_SELECT_INVERSE;
 
-    my $command = "$difftool -diffskyfile -dbname $imagedb";
+    my $command = "$difftool -dbname $imagedb";
     
     if ($byid) {
-        $command .= " -diff_id $id";
+        $command .= " -listrun -diff_id $id";
     } else {
-        $command .= " -diff_skyfile_id $id";
+        $command .= " -diffskyfile -diff_skyfile_id $id";
     }
     $command .= " -skycell_id $skycell_id" if $skycell_id;
 
-    my $images = runToolAndParse($command, $verbose);
-
-    my $n = @$images;
+    my $output = runToolAndParse($command, $verbose);
+
+    my $n = @$output;
     if (!$byid && ($n > 1)) {
         die ("difftool returned an unexpected number of diffskyfiles: $n");
     } elsif ($n == 0) {
         return undef;
+    }
+
+    my $images;
+    if ($byid) {
+        $images = selectComponents($ipprc, $imagedb, 'byid', 'diff', $rowList, $output, $verbose);
+    } else {
+        $images = $output;
     }
 

trunk/ippTasks/pstamp.pro

@@ -524 +524 @@
 
     periods     -poll $LOADPOLL
-    # XXX: create a macro for this
-    periods     -exec 30
+    # XXX: create a macro for this time
+    periods     -exec 10
 #    periods     -exec $LOADEXEC
-    periods     -timeout 30
+    periods     -timeout 300
     npending    1
 

trunk/ippTools/share/Makefile.am

@@ -114 +114 @@
      difftool_pendingcleanupskyfile.sql \
      difftool_revertdiffskyfile_delete.sql \
-     difftool_revertdiffskyfile_update.sql \
+     difftool_revertdiffskyfile_updated.sql \
+     difftool_setskyfiletoupdate.sql \
      difftool_skyfile.sql \
      difftool_todiffskyfile.sql \
@@ -223 +224 @@
      pstamptool_completedreq.sql \
      pstamptool_datastore.sql \
+     pstamptool_getdependent.sql \
      pstamptool_listjob.sql \
      pstamptool_pendingdependent.sql \

trunk/ippTools/share/pstamptool_pendingdependent.sql

@@ -4 +4 @@
 JOIN pstampRequest USING(req_id)
 WHERE pstampDependent.state = 'new'
+    AND pstampDependent.fault = 0
     AND pstampJob.state = 'run'
+    AND pstampRequest.state = 'run'

trunk/ippTools/share/pstamptool_revertdependent.sql

@@ -3 +3 @@
     JOIN pstampRequest USING(req_id)
 SET pstampDependent.fault = 0
+WHERE pstampDependent.state = 'new'

trunk/ippTools/src/pstamptool.c

@@ -530 +530 @@
 
     PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-fault", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fault", "fault", "==");
     PXOPT_COPY_STR(config->args, where, "-state", "pstampRequest.state", "==");
     pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE");
@@ -967 +967 @@
     PXOPT_LOOKUP_STR(component,   config->args, "-component",  true, false);
     PXOPT_LOOKUP_STR(imagedb,     config->args, "-imagedb",  true, false);
-    PXOPT_LOOKUP_STR(rlabel,      config->args, "-rlabel",  false, false);
+    PXOPT_LOOKUP_STR(rlabel,      config->args, "-rlabel",  true, false);
     PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false);
     PXOPT_LOOKUP_BOOL(no_create,  config->args, "-no_create", false);
@@ -983 +983 @@
     }
 
-    psString query = pxDataGet("pstamptool_pendingdependent.sql");
+    psString query = pxDataGet("pstamptool_getdependent.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -1186 +1186 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-fault", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fault", "pstampDependent.fault", "==");
     PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
     pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "==");
@@ -1201 +1201 @@
     }
 
-    psString whereClause = psDBGenerateWhereSQL(where, NULL);
-    psStringAppend(&query, " %s", whereClause);
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
     psFree(whereClause);
     psFree(where);

trunk/ippTools/src/pstamptoolConfig.c

@@ -177 +177 @@
     psMetadata *revertdependentArgs = psMetadataAlloc();
     psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent", 0);
-    psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault",  0,   "new value for fault", 0);
+    psMetadataAddStr(revertdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK,   "define label for dependent ", NULL);
+    psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault",  0,   "define fault", 0);
 
     // -pendingdependent

trunk/pstamp/scripts/pstamp_checkdependent.pl

@@ -270 +270 @@
 
         if ($good_to_go) {
+            # the reason we defer setting the warp to update is so that we can handle error conditions at previous
+            # stages more easily.
             my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
             $command .= " -set_label $rlabel" if $rlabel;
@@ -341 +343 @@
         my $skycell_id = $skycell->{skycell_id};
 
-        # put this in a module somewhere
+        # XXX: put this in a module somewhere
         my $IPP_DIFF_MODE_WARP_WARP   = 1;
         my $IPP_DIFF_MODE_WARP_STACK  = 2;
@@ -347 +349 @@
         my $IPP_DIFF_MODE_STACK_STACK = 4;
         if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {
-            # check the state of the stack
+            # check the state of the template stack
             my $command = "$stacktool -sumkskyfile -stack_id $skycell->{stack2}";
             my $stack = runToolAndParseExpectOne($command, $verbose);
@@ -361 +363 @@
             # now check the warp
             $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
-            my $warp = runToolAndExpectOne($command, $verbose);
-            my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR)
-                if !$warp;
-        }
-
-        my $good_to_go = 1;
-        my @chipsToUpdate;
-        my $chip_id;
-        my $warp_id;
-        my $data ;  #XXXX
-        foreach my $chip (@$data) {
-            $chip_id = $chip->{chip_id};
-            if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
-                $good_to_go = 0;
-                push @chipsToUpdate, $chip;
-            } else {
-                # this chip is good to go
-            }
-        }
-
-        if ($good_to_go) {
-            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
-            $command .= " -set_label $rlabel" if $rlabel;
-
-            if (!$no_update) {
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                            run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+            my $warp = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
+
+            if ($warp->{data_state} ne 'full') {
+                return queue_update_warp($warp, 0, $rlabel, $need_magic);
+            }
+            # warps are ready fall through and queue the diff update
+        } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) {
+            my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
+            my $warp1 = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1;
+
+            if ($warp1->{data_state} ne 'full') {
+                my $status = queue_update_warp($warp1, 0, $rlabel, $need_magic);
+                if ($status) {
+                    return $status;
                 }
-            } else {
-                print "skipping $command\n";
-            }
-        } elsif (scalar @chipsToUpdate > 0) {
-            return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic);
-        }
-    } else {
-        my $warpRun = $metadata;
-        my $warp_id = $warpRun->{warp_id};
-        my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}";
-        my $data = runToolAndParse($command, $verbose);
-        my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR)
-            if !$data or scalar @$data != 1;
-
-        my $chipRun = $data->[0];
-
-        my $chipRunState = $chipRun->{state};
-        if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) {
-            print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n";
-            faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE);
-            return 0;
-        }
-        my $warpRunState = $warpRun->{state};
-        if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) {
-            # The inputs and outputs are ready. Queue the warpRun for update.
-
-            # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update'
-            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id";
-            $command .= " -set_label $rlabel" if $rlabel;
-
-            if (!$no_update) {
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                            run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
-                }
-            } else {
-                print "skipping $command\n";
-            }
-        } elsif ($chipRunState eq 'cleaned' or 
-            (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) {
-            my $data = [$chipRun];
-            return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic);
-        }
-    }
-
+            }
+            $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
+            my $warp2 = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2;
+
+            if ($warp2->{data_state} ne 'full') {
+                return queue_update_warp($warp2, 0, $rlabel, $need_magic);
+            }
+            # fall through to queue the diff update
+        } else {
+            # XXX: check stack stack
+            print STDERR "unexpected diff_mode found: $diff_mode\n";
+            return $PS_EXIT_PROG_ERROR;
+        }
+
+        my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+    } else {
+        # XXX todo whole_run
+        return $PS_EXIT_PROG_ERROR;
+    }
     # return value may be the return status of script so zero is good
     return 0;
@@ -593 +567 @@
     carp $msg;
 
-    my $command = "$pstamptool -updatedepedent -fault $fault -dep_id $dep_id";
+    my $command = "$pstamptool -updatedependent -fault $fault -dep_id $dep_id";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -600 +574 @@
                     run(command => $command, verbose => $verbose);
         unless ($success) {
-            my_die("failed to set pstampDependent.fault for dep_id: $dep_id",
-                $PS_EXIT_UNKNOWN_ERROR);
+            carp "$cmd failed";
         }
     } else {

trunk/pstamp/scripts/pstampparse.pl

@@ -536 +536 @@
     # XXX: this code is repeated in queueGetImageJobs we should encapsulate it in a subroutine and share it
     if ($stage ne 'raw') {
-        # not ready to handle diff update yet. May never support stack
-        my $allow_wait_for_update = (($stage ne 'stack') and ($stage ne 'diff'));
+        # updates for stack stage not supported yet
+        my $allow_wait_for_update = ($stage ne 'stack');
         my $run_state = $image->{state};
         my $data_state = $image->{data_state};
@@ -548 +548 @@
             $newState = 'stop';
             $fault = $PSTAMP_GONE;
-        } elsif ($allow_wait_for_update and 
-                (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0)))) {
+        } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) {
             if ($stage eq 'chip') {
                 my $burntool_state = $image->{burntool_state};
@@ -556 +555 @@
                     $fault = $PSTAMP_NOT_AVAILABLE;
                 }
+            } 
+            if (!$allow_wait_for_update) {
+                print STDERR "wait for update not supported for stage $stage yet\n";
+                $newState = 'stop';
+                $fault = $PSTAMP_NOT_AVAILABLE;
             }
             if (!$fault) {