Changeset 26030

Timestamp:

Nov 4, 2009, 11:34:42 AM (17 years ago)

Author:

bills

Message:

add tasks and scripts that execute magicdstool -definebyquery and disttool -definebyquery

Location:

trunk

Files:

• ippScripts/Build.PL (modified) (2 diffs)
• ippScripts/scripts/dist_defineruns.pl (moved) (moved from trunk/ippScripts/scripts/dist_queue_runs.pl ) (5 diffs, 1 prop)
• ippScripts/scripts/magic_destreak_defineruns.pl (added)
• ippTasks/survey.pro (modified) (3 diffs)

trunk/ippScripts/Build.PL

@@ -60 +60 @@
         scripts/magic_destreak_revert.pl
         scripts/magic_destreak_cleanup.pl
+        scripts/magic_destreak_defineruns.pl
         scripts/ippdb.pl
         scripts/ipp_cleanup.pl
@@ -87 +88 @@
         scripts/dist_advancerun.pl
         scripts/dist_make_fileset.pl
-        scripts/dist_queue_runs.pl
+        scripts/dist_defineruns.pl
         scripts/receive_source.pl
         scripts/receive_fileset.pl

trunk/ippScripts/scripts/dist_defineruns.pl

@@ -1 +1 @@
 #!/usr/bin/env perl
+#
+# dist_queue_runs.pl : run disttool -definebyquery for the various stages or one stage
+#
 
 use Carp;
@@ -40 +43 @@
 my ($stage, $stage_limit, $dist_root, $no_magic);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
+my @labels;
 
 GetOptions(
+           'stage=s'        => \$stage,      # stage to queue
+           'label=s'        => \@labels,     # labels
+           'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
            'dist_root=s'    => \$dist_root,  # root of distribution work area
-           'stage=s'        => \$stage,      # stage to queue
-           'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
-           'no_magic'       => \$no_magic,   # queue runs without requiring magic
+           'no_magic'       => \$no_magic,   # queue runs without requiring magic (for testing only)
            'dbname=s'       => \$dbname,     # Database name
            'verbose'        => \$verbose,    # Print stuff?
@@ -54 +59 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-if (0) {
-pod2usage( -msg => "Required options: --stage",
+pod2usage( -msg => "Required options: --label",
            -exitval => 3) unless
-    defined $stage;
-}
+    (scalar @labels > 0);
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -67 +70 @@
 }
 
-my ($day, $month, $year) = (localtime)[3,4,5];
+my ($day, $month, $year) = (gmtime)[3,4,5];
 my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
 
@@ -82 +85 @@
 }
 
-# XXX: how shall we deal with labels?
-my $label = 'proc';
-
 foreach my $stage (@stages) {
-    my $command = "$disttool -definebyquery -stage $stage -workdir $workdir";
-    $command .= " -no_magic" if $no_magic;
-    $command .= " -dry_run" if $no_update;
-    $command .= " -limit $stage_limit" if $stage_limit;
-    $command .= " -set_label $label";
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform $command error_code: $error_code", $error_code);
+    foreach my $label (@labels) {
+        my $command = "$disttool -definebyquery -stage $stage -workdir $workdir -label $label";
+        $command .= " -no_magic" if $no_magic;
+        $command .= " -dry_run" if $no_update;
+        $command .= " -limit $stage_limit" if $stage_limit;
+        $command .= " -set_label $label";
+        $command .= " -dbname $dbname" if defined $dbname;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command error_code: $error_code", $error_code);
+        }
+        # display the output from the command
+        print STDERR join "", @$stdout_buf if $verbose;
     }
-    # display the output from the command
-    print STDERR join "", @$stdout_buf if $verbose;
 }
 
+if (0) {
+# notyet
 # queue rcRuns for any distRuns that have completed and have interested destinations
-{
     my $command = "$disttool -queuercrun";
     $command .= " -dbname $dbname" if defined $dbname;

trunk/ippTasks/survey.pro

@@ -12 +12 @@
   book create SURVEY_MAGIC
 end
+book getbook SURVEY_DESTREAK -var isBook
+if ("$isBook" == "NULL")
+  book create SURVEY_DESTREAK
+end
+book getbook SURVEY_DIST -var isBook
+if ("$isBook" == "NULL")
+  book create SURVEY_DIST
+end
 
 $SURVEY_DIFF_DB = 0
 $SURVEY_MAGIC_DB = 0
+$SURVEY_DESTREAK_DB = 0
+$SURVEY_DIST_DB = 0
 
 $SURVEY_EXEC = 120
@@ -74 +84 @@
   end
   book listbook SURVEY_MAGIC
+end
+
+# user functions to manipulate destreak labels
+macro survey.add.destreak
+  if ($0 != 3)
+    echo "USAGE: survey.add.destreak (label) (workdir base)"
+    break
+  end
+  book newpage SURVEY_DESTREAK $1
+  book setword SURVEY_DESTREAK $1 WORKDIR $2
+  book setword SURVEY_DESTREAK $1 STATE PENDING
+end
+
+macro survey.del.destreak
+  if ($0 != 2)
+    echo "USAGE: survey.del.destreak (label)"
+    break
+  end
+  book delpage SURVEY_DESTREAK $1
+end
+
+macro survey.show.destreak
+  if ($0 != 1)
+    echo "USAGE: survey.show.destreak"
+    break
+  end
+  book listbook SURVEY_DESTREAK
+end
+
+# user functions to manipulate dist labels
+macro survey.add.dist
+  if ($0 != 2)
+    echo "USAGE: survey.add.dist (label)"
+    break
+  end
+  book newpage SURVEY_DIST $1
+  book setword SURVEY_DIST $1 STATE PENDING
+end
+
+macro survey.del.dist
+  if ($0 != 2)
+    echo "USAGE: survey.del.dist (label)"
+    break
+  end
+  book delpage SURVEY_DIST $1
+end
+
+macro survey.show.dist
+  if ($0 != 1)
+    echo "USAGE: survey.show.dist"
+    break
+  end
+  book listbook SURVEY_DIST
 end
 
@@ -229 +292 @@
 end
 
+task survey.destreak
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  active false
+
+  stdout $LOGDIR/survey.destreak.log
+  stderr $LOGDIR/survey.destreak.log
+
+  # generate destreak runs
+  task.exec
+    book npages SURVEY_DESTREAK -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_DESTREAK 0 -var label -key STATE NEW
+    if ("$label" == "NULL")
+      # All labels have been done --- reset
+#      echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_DESTREAK $i -var label
+        book setword SURVEY_DESTREAK $label STATE NEW
+      end
+      book getpage SURVEY_DESTREAK 0 -var label -key STATE NEW
+
+      # Select different database
+      $SURVEY_DESTREAK_DB ++
+      if ($SURVEY_DESTREAK_DB >= $DB:n)
+        set $SURVEY_DESTREAK_DB = 0
+      end
+    end
+
+    book setword SURVEY_DESTREAK $label STATE DONE
+    book getword SURVEY_DESTREAK $label WORKDIR -var workdir
+  
+    $run = magic_destreak_defineruns.pl --label $label --workdir $workdir/$label/
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run -dbname $DB:$SURVEY_MAGIC_DB
+      option $DB:$SURVEY_MAGIC_DB
+    end
+    
+#    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task survey.dist
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.dist.log
+  stderr $LOGDIR/survey.dist.log
+
+  # generate distribution runs
+  task.exec
+    book npages SURVEY_DIST -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_DIST 0 -var label -key STATE NEW
+    if ("$label" == "NULL")
+      # All labels have been done --- reset
+#      echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_DIST $i -var label
+        book setword SURVEY_DIST $label STATE NEW
+      end
+      book getpage SURVEY_DIST 0 -var label -key STATE NEW
+
+      # Select different database
+      $SURVEY_DIST_DB ++
+      if ($SURVEY_DIST_DB >= $DB:n)
+        set $SURVEY_DIST_DB = 0
+      end
+    end
+
+    book setword SURVEY_DIST $label STATE DONE
+  
+    # note workdir is set by the script based on site.config
+    $run = dist_defineruns.pl --label $label
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run --dbname $DB:$SURVEY_DIST_DB
+      option $DB:$SURVEY_DIST_DB
+    end
+    
+#    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end