Changeset 19545

Timestamp:

Sep 12, 2008, 4:20:57 PM (18 years ago)

Author:

eugene

Message:

convert chiphost tables to ipphosts tables, add lookups for skycellID to host, add in the redirect-output option to stack and diff

Location:

trunk

Files:

• ippScripts/scripts/diff_skycell.pl (modified) (4 diffs)
• ippScripts/scripts/stack_skycell.pl (modified) (5 diffs)
• ippTasks/Makefile.am (modified) (1 diff)
• ippTasks/diff.pro (modified) (1 diff)
• ippTasks/ipphosts.manoa.config (added)
• ippTasks/ipphosts.mhpcc.config (added)
• ippTasks/pantasks.pro (modified) (6 diffs)
• ippTasks/site.manoa.pro (modified) (2 diffs)
• ippTasks/site.mhpcc.pro (modified) (3 diffs)
• ippTasks/stack.pro (modified) (1 diff)
• ippTasks/warp.pro (modified) (2 diffs)

trunk/ippScripts/scripts/diff_skycell.pl

@@ -25 +25 @@
 use Pod::Usage qw( pod2usage );
 
-my ($diff_id, $dbname, $threads, $outroot, $verbose, $no_update, $no_op);
+my ($diff_id, $dbname, $threads, $outroot, $verbose, $no_update, $no_op, $redirect);
 GetOptions(
     'diff_id|d=s'       => \$diff_id, # Diff identifier
@@ -34 +34 @@
     'no-update'         => \$no_update, # Don't update the database?
     'no-op'             => \$no_op, # Don't do any operations?
+    'redirect-output'   => \$redirect,
 ) or pod2usage( 2 );
 
@@ -42 +43 @@
           ) unless defined $diff_id
     and defined $outroot;
+
+# XXX camera is not known here; cannot use filerules...
+# my $logDest = $ipprc->filename("LOG.EXP", $outroot);
+
+my $logDest = "$outroot.log";
+$ipprc->redirect_output($logDest) if $redirect;
 
 my $STATS =
@@ -183 +190 @@
 my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot);
 my $traceDest = $ipprc->filename("TRACE.EXP", $outroot);
-my $logDest = $ipprc->filename("LOG.EXP", $outroot);
 
 # Perform subtraction

trunk/ippScripts/scripts/stack_skycell.pl

@@ -28 +28 @@
 use Pod::Usage qw( pod2usage );
 
-my ($stack_id, $dbname, $outroot, $debug, $threads, $verbose, $no_update, $no_op, $save_temps);
+my ($stack_id, $dbname, $outroot, $debug, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps);
 GetOptions(
     'stack_id|d=s'      => \$stack_id, # Stack identifier
@@ -38 +38 @@
     'no-update'         => \$no_update, # Don't update the database?
     'no-op'             => \$no_op, # Don't do any operations?
+    'redirect-output'   => \$redirect,
     'save-temps'        => \$save_temps, # Save temporary files?
 ) or pod2usage( 2 );
@@ -47 +48 @@
           ) unless defined $stack_id
     and defined $outroot;
+
+# XXX camera is not known here; cannot use filerules...
+# my $logDest = $ipprc->filename("LOG.EXP", $outroot);
+
+my $logDest = "$outroot.log";
+$ipprc->redirect_output($logDest) if $redirect;
 
 my $STATS =
@@ -97 +104 @@
 my $camera;                     # Camera
 foreach my $file (@$files) {
-    # skip warps which are speicified as 'ignored'
+    # skip warps which are specified as 'ignored'
     if ($file->{ignored}) { next; }
     if (defined $tess_id) {
@@ -166 +173 @@
 my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot);
 my $traceDest = $ipprc->filename("TRACE.EXP", $outroot);
-my $logDest = $ipprc->filename("LOG.EXP", $outroot);
 my $configuration = $ipprc->filename("PPSTACK.CONFIG", $outroot);
 

trunk/ippTasks/Makefile.am

@@ -21 +21 @@
 
 other_files = \
-        chiphosts.manoa.config \
-        chiphosts.mhpcc.config \
+        ipphosts.manoa.config \
+        ipphosts.mhpcc.config \
         site.manoa.pro \
         site.mhpcc.pro \

trunk/ippTasks/diff.pro

@@ -142 +142 @@
     book getword diffSkyfile $pageName dbname -var DBNAME
 
-    ### XXX Instead of setting these by camera, need to set by skycell_id
-    ### XXX change these to use the --redirect scheme
-#    set.host.for.camera $CAMERA $DIFF_ID
-#    set.workdir.by.camera $CAMERA $DIFF_ID $WORKDIR_TEMPLATE $default_host WORKDIR
-    host anyhost
-    $WORKDIR = $WORKDIR_TEMPLATE
+    # set the host and workdir based on the skycell hash
+    set.host.for.skycell $SKYCELL_ID
+    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    # XXX old code:
+    # host anyhost
+    # $WORKDIR = $WORKDIR_TEMPLATE
 
     basename $TESS_DIR -var TESS_ID
     sprintf outroot "%s/%s/%s/%s.%s.dif.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $DIFF_ID
 
-    ## generate output log based on filerule (convert the URI to a PATH)
-    $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $DIFF_ID --basename $outroot`
-    if ("$logfile" == "") 
-      echo "WARNING: logfile not defined in diff.skycell.run"
-      break
-    end
-
-    stdout $logfile
-    stderr $logfile
-    dirname $logfile -var outpath
-    mkdir $outpath
-
-    $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --outroot $outroot
+    stdout $LOGDIR/diff.skycell.log
+    stderr $LOGDIR/diff.skycell.log
+
+    $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --outroot $outroot --redirect-output
     add_standard_args run
 

trunk/ippTasks/pantasks.pro

@@ -350 +350 @@
   end
 
-  # missing camera and/or chiphosts table results in host = NULL
-  book getword chiphosts $camera $classID -var host
+  # missing camera and/or ipphosts table results in host = NULL
+  book getword ipphosts $camera $classID -var host
 
   if ("$host" == "NULL")
@@ -360 +360 @@
 end
 
-macro set.volume.by.camera
-  if ($0 != 3)
-    echo "USAGE: set.volume.by.camera (camera) (class_id) (template) (default) (varname)"
-    echo " uses global variable NEBULOUS to determine if nebulous is active"
-    break
-  end
-
-  local host default template camera classID varname
-  $camera = $1
-  $classID = $2
-  $template = $3
-  $default = $4
-  $varname = $5
-
-  # missing camera and/or chiphosts table results in host = NULL
-  book getword chiphosts $camera $classID -var host
-
-  if (not($NEBULOUS))
-    $$varname = "NULL"
-    return
-  end
-
-  if ("$host" == "NULL") 
-    $$varname = $default
-    return
-  end
-
-  strsub $template @HOST@ $host -var $varname
+macro set.workdir.by.skycell
+  if ($0 != 5)
+    echo "USAGE: set.workdir.by.skycell (skycellID) (template) (default) (varname)"
+    break
+  end
+
+  local host default template skycellID varname count
+  $skycellID = $1
+  $template = $2
+  $default = $3
+  $varname = $4
+
+  if ("$template" == "NULL")
+    $varname = $workdir_template
+    echo "WARNING: WORKDIR template not set.  Defaulting to $workdir_template"
+    break
+  end
+
+  # get the folding count for this camera  
+  book getword ipphosts skycell count -var count
+  if ("$count" == "NULL")
+    strsub $template @HOST@ $default -var $varname
+    return
+  end    
+
+  strhash $skycellID $count -var skyhash
+  sprintf skyname "sky%02d" $skyhash
+
+  # missing ipphosts table results in host = NULL
+  book getword ipphosts skycell $skyname -var host
+
+  if ("$host" == "NULL")
+    strsub $template @HOST@ $default -var $varname
+  else
+    strsub $template @HOST@ $host -var $varname
+  end
 end
 
@@ -396 +404 @@
   end
 
+  local host
+
   if (not($PARALLEL))
     host local
@@ -401 +411 @@
   end
 
-  # missing camera and/or chiphosts table results in host = NULL
-  book getword chiphosts $1 $2 -var host
+  # missing camera and/or ipphosts table results in host = NULL
+  book getword ipphosts $1 $2 -var host
 
   if ("$host" == "NULL")
@@ -411 +421 @@
 end
 
+macro set.host.for.skycell
+  if ($0 != 2)
+    echo "USAGE: set.host.for.skycell (skycellID)"
+    break
+  end
+
+  local skycellID varname count host skyname skyhash
+  $skycellID = $1
+
+  # get the folding count for this camera  
+  book getword ipphosts skycell count -var count
+  if ("$count" == "NULL")
+    host anyhost
+    return
+  end    
+
+  strhash $skycellID $count -var skyhash
+  sprintf skyname "sky%02d" $skyhash
+
+  # missing ipphosts table results in host = NULL
+  book getword ipphosts skycell $skyname -var host
+
+  if ("$host" == "NULL")
+    host anyhost
+  else
+    host $host
+  end
+end
+
 macro get.host.for.camera
   if ($0 != 3)
@@ -422 +461 @@
   end
 
-  # missing camera and/or chiphosts table results in host = NULL
-  book getword chiphosts $1 $2 -var host
+  # missing camera and/or ipphosts table results in host = NULL
+  book getword ipphosts $1 $2 -var host
 
   if ("$host" == "NULL")

trunk/ippTasks/site.manoa.pro

@@ -4 +4 @@
 ## 1) the machines used for processing
 ## 2) the default_host and workdir_template variables
-## 3) the chiphost table
+## 3) the ipphost table
 
 macro init.cluster.po
@@ -48 +48 @@
 
 macro init.site
-  queueload tmp -x "cat $MODULES:0/chiphosts.manoa.config"
-  ipptool2book tmp chiphosts -key camera
+  queueload tmp -x "cat $MODULES:0/ipphosts.manoa.config"
+  ipptool2book tmp ipphosts -key camera
 end

trunk/ippTasks/site.mhpcc.pro

@@ -4 +4 @@
 ## 1) the machines used for processing
 ## 2) the default_host and workdir_template variables
-## 3) the chiphost table
+## 3) the ipphost table
 
 macro init.cluster.mhpcc
@@ -59 +59 @@
   init.copy.mhpcc on
 
-  queueload tmp -x "cat $MODULES:0/chiphosts.mhpcc.config"
-  ipptool2book tmp chiphosts -key camera
+  queueload tmp -x "cat $MODULES:0/ipphosts.mhpcc.config"
+  ipptool2book tmp ipphosts -key camera
 end
 
@@ -66 +66 @@
   init.copy.mhpcc on
 
-  queueload tmp -x "cat $MODULES:0/chiphosts.mhpcc.config"
-  ipptool2book tmp chiphosts -key camera
+  queueload tmp -x "cat $MODULES:0/ipphosts.mhpcc.config"
+  ipptool2book tmp ipphosts -key camera
 end

trunk/ippTasks/stack.pro

@@ -142 +142 @@
     book getword stackSumSkyfile $pageName dbname -var DBNAME
 
-    ### XXX Instead of setting these by camera, need to set by skycell_id
-#    set.host.for.camera $CAMERA $STACK_ID
-#    set.workdir.by.camera $CAMERA $STACK_ID $WORKDIR_TEMPLATE $default_host WORKDIR
-    host anyhost
-    $WORKDIR = $WORKDIR_TEMPLATE
+    # set the host and workdir based on the skycell hash
+    set.host.for.skycell $SKYCELL_ID
+    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    # XXX old code:
+    # host anyhost
+    # $WORKDIR = $WORKDIR_TEMPLATE
 
     basename $TESS_DIR -var TESS_ID
     sprintf outroot "%s/%s/%s/%s.%s.stk.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $STACK_ID
 
-    ## generate output log based on filerule (convert the URI to a PATH) (use STACK_ID?)
-#    $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $STACK_ID --basename $outroot`
-    $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot`
-    if ("$logfile" == "") 
-      echo "WARNING: logfile not defined in stack.pro:135"
-      break
-    end
-
-    stdout $logfile
-    stderr $logfile
-    dirname $logfile -var outpath
-    mkdir $outpath
-
-    $run = stack_skycell.pl --threads @MAX_THREADS@ --stack_id $STACK_ID --outroot $outroot
+    stdout $LOGDIR/stack.skycell.log
+    stderr $LOGDIR/stack.skycell.log
+
+    $run = stack_skycell.pl --threads @MAX_THREADS@ --stack_id $STACK_ID --outroot $outroot --redirect-output
     add_standard_args run
 

trunk/ippTasks/warp.pro

@@ -158 +158 @@
     book getword warpInputExp $pageName exp_tag -var EXP_TAG
 
-    ### XXX Instead of setting these by camera, need to set by skycell_id
+    # set the host and workdir (default)
     set.host.for.camera $CAMERA $WARP_ID
     set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR
@@ -273 +273 @@
     book getword warpPendingSkyCell $pageName exp_tag -var EXP_TAG
 
-    ### XXX Instead of setting these by camera, need to set by skycell_id
-    set.host.for.camera $CAMERA $WARP_ID
-    set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR
-
-    # XXX we are supplied TESS_DIR, strip off path to generate TESS_ID
-    # basename $TESS_DIR -var TESS_ID
-    # sprintf outroot "%s/%s/%s/%s.%s.wrp.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $WARP_ID
+    # set the host and workdir based on the skycell hash
+    set.host.for.skycell $SKYCELL_ID
+    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
 
     ## generate outroot specific to this exposure