Changeset 19381

Timestamp:

Sep 4, 2008, 2:30:59 PM (18 years ago)

Author:

eugene

Message:

adding flatcorr simtest suites, mods to allow camera and sequence to be supplied by user

Location:

trunk/ippTasks

Files:

• Makefile.am (modified) (1 diff)
• automate.pro (modified) (2 diffs)
• notes.txt (modified) (1 diff)
• simtest.basic.auto (added)
• simtest.flatcorr.auto (modified) (3 diffs)
• simtest.flatcorr.config (added)
• simtest.pro (modified) (4 diffs)

trunk/ippTasks/Makefile.am

@@ -23 +23 @@
         chiphosts.manoa.config \
         chiphosts.mhpcc.config \
-        simtest.basic.config \
         site.manoa.pro \
         site.mhpcc.pro \
         simtest.pro \
-        simtest.auto \
-        simtest.det \
-        isp.det
+        simtest.basic.config \
+        simtest.basic.auto \
+        simtest.flatcorr.config \
+        simtest.flatcorr.auto
 
 pantasksdir = $(datadir)/pantasks

trunk/ippTasks/automate.pro

@@ -23 +23 @@
 
 macro automate.load
-  if ($0 != 3)
-    echo "USAGE: automate.load (filename) (dbname)"
+  if ($0 != 4)
+    echo "USAGE: automate.load (filename) (camera) (dbname)"
     break
   end
@@ -32 +32 @@
 
   ## interpolate standard values
-  queuesubstr tmp @DBNAME@ $2
+  queuesubstr tmp @CAMERA@ $2
+  queuesubstr tmp @DBNAME@ $3
   queuesubstr tmp @CWD@ $cwd
 

trunk/ippTasks/notes.txt

@@ -34 +34 @@
    * add an entry to ppMerge/src/ppMergeArguments.c
    * add an entry to ippconfig/recipes/ppMerge.config
+
    * add an entry to ippconfig/recipes/rejections.config
-   
+   * add a section to ippconfig/recipes/reductionClasses.mdc
+
+   * ippScripts/scripts/detrend_norm_apply.pl : entry in DETTYPE table
+   * ippScripts/scripts/detrend_norm_calc.pl : entry in NORMALIZE table
+   * ippScripts/scripts/detrend_resid_imfile.pl : entry in DETRENDS table
+   * ippScripts/scripts/detrend_stack.pl : entry in FILERULES and STATRECIPES tables
+   * ippScripts/scripts/detrend_resid_imfile.pl : entry in DETRENDS table
+
   * need a block in simtest.auto (or equiv) to check for when the chip processing / cam processing is complete
 
   * need to modify the detselect -inst arguments in simtest.auto (use @INST@ or equiv)
+
+  * reductions currently being used:
+    ** CHIP -- chip-level analysis
+    ** JPEG.BIN1, JPEG.BIN2 -- camera jpeg terms
+    ** JPEG_BIN1_IMAGE_(dettype), JPEG_BIN2_IMAGE_(dettype) 
+    ** JPEG_BIN1_RESID_(dettype), JPEG_BIN2_RESID_(dettype) 
+    ** (dettype)_STACK, 
+    ** (dettype)_PROCESS, 
+    ** FAKEPHOT
 
 2008.01.22

trunk/ippTasks/simtest.flatcorr.auto

@@ -5 +5 @@
 # success on CHECK -> LAUNCH
 # failure on BLOCK -> CHECK
+
+# this is a simplified automate script which only models BIAS and FLAT.  
+# generate appropriate simulated data with the FLATCORR recipe
 
 # wait until 20 bias frames are registered, then define a new bias run
@@ -22 +25 @@
   block      STR "dettool -runs -active -det_type DARK -dbname @DBNAME@"
 END
- 
+
 # wait until the master dark frame is registered, then define a new shutter run
 automate METADATA
@@ -31 +34 @@
 END
 
-# wait until the master shutter frame is registered, then define a new flat-r run
+# wait until the master bias frame is registered, then define a new flat-r run
 automate METADATA
-  name       STR FLAT-r
+  name       STR RAWFLAT-r
   check      STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@"
-  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@"
-  block      STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@"
+  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT_RAW -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type FLAT_RAW -filter r -dbname @DBNAME@"
 END
 
-# wait until the master shutter frame is registered, then define a new flat-i run
+# wait until the master bias frame is registered, then define a new flat-i run
 automate METADATA
-  name       STR FLAT-i
+  name       STR RAWFLAT-i
   check      STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@"
-  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type FLAT -select_filter i -dbname @DBNAME@"
-  block      STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@"
+  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT_RAW -filter i -select_exp_type FLAT -select_filter i -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type FLAT_RAW -filter i -dbname @DBNAME@"
 END
 
-# wait until the master flat-r frame is registered, then unblock the r-band data for chip-level processing
-automate METADATA
-  name       STR OBJECT-i
-  check      STR "detselect -dbname @DBNAME@ -search    -inst @CAMERA@ -filter i -det_type FLAT"
-  launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc"
-  block      STR NONE
-END
+## # wait until the master flat-r frame is registered, then unblock the r-band data for chip-level processing
+## automate METADATA
+##   name       STR OBJECT-i
+##   check      STR "detselect -dbname @DBNAME@ -search    -inst @CAMERA@ -filter i -det_type FLAT"
+##   launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc"
+##   block      STR NONE
+## END
+## 
+## # wait until the master flat-i frame is registered, then unblock the i-band data for chip-level processing
+## automate METADATA
+##   name       STR OBJECT-r
+##   check      STR "detselect -dbname @DBNAME@ -search    -inst @CAMERA@ -filter r -det_type FLAT"
+##   launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc"
+##   block      STR NONE
+## END
+## 
+## # wait until the r-band is done with camera, then define a new flat-correction run
+## automate METADATA
+##   name       STR FLATCORR-i
+##   check      STR "camtool -dbname @DBNAME@ -search -inst @CAMERA@ -filter r"
+##   ncheck     S32 8
+##   launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc"
+##   block      STR NONE
+## END
 
-# wait until the master flat-i frame is registered, then unblock the i-band data for chip-level processing
-automate METADATA
-  name       STR OBJECT-r
-  check      STR "detselect -dbname @DBNAME@ -search    -inst @CAMERA@ -filter r -det_type FLAT"
-  launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc"
-  block      STR NONE
-END
-
-# wait until the r-band is done with camera, then define a new flat-correction run
-automate METADATA
-  name       STR FLATCORR-i
-  check      STR "camtool -dbname @DBNAME@ -search -inst @CAMERA@ -filter r"
-  ncheck     S32 8
-  launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc"
-  block      STR NONE
-END
-
+## flatcorr -definebyquery -dbname eamtest -filter r -set_workdir /data/alala.0/eugene/swtests/simtest/test.f1/work -set_reduction FLATCORR -set_dvodb /data/alala.0/eugene/swtests/simtest/test.f1/catdir.flatcorr -set_region 270.6,270.9:-23.8,-23.5 -set_filter r

trunk/ippTasks/simtest.pro

@@ -1 +1 @@
 ## simtest.pro : automatic full-suite simulated IPP test : -*- sh -*-
 
+## there are four features which define a simtest run:
+## the camera : any camera can be used, the sim... cameras are defined to be quick tests
+## the sequence : this file defines the set of observations generated
+## the auto : the automation file defines the sequence of analysis performed
+## the recipe : the ppsim recipe defines the features which are included in the generate data
+
+## these are not completely independent: a certain sequence may be
+## required to yield a sensible result for a specific auto, and a
+## specific recipe may also be needed
+
+if ($?PPSIM_RECIPE == 0) 
+  $PPSIM_RECIPE = default
+end
+if ($?SIMTEST_CAMERA == 0)
+  $SIMTEST_CAMERA = SIMTEST
+end
+if ($?SIMTEST_SEQUENCE == 0)
+  $SIMTEST_SEQUENCE = simtest.basic.config
+end
+if ($?SIMTEST_AUTO == 0)
+  $SIMTEST_AUTO = simtest.basic.auto
+end
+if ($?SIMTEST_THREADS == 0) 
+  $SIMTEST_THREADS = 0
+end
+
 macro simtest
-  if ($0 != 6)
-    echo "USAGE: simtest (testroot) (dbname) (hostname) (threads) (init)"
-    echo " if (init) == 0, just run the analysis tasks"
-    echo " if (init) == 1, (re)create the database and (re)inject the images"
-    echo " if (init) == 2, restart completely from scratch"
+  if ($0 != 4)
+    echo "USAGE: simtest (dbname) (hostname) (init)"
+    echo " if (init) == run    : just run the analysis tasks"
+    echo " if (init) == inject : (re)create the database and (re)inject the images"
+    echo " if (init) == new    : restart completely from scratch"
+    echo ""
+    echo "there are additional options which may be specified by setting the following variable names:" 
+    echo "  PPSIM_RECIPE     : define the recipe to use when generating the fake data"
+    echo "  SIMTEST_CAMERA   : define an alternate camera (otherwise uses entry in sequence file)"
+    echo "  SIMTEST_SEQUENCE : define the set of observations generated (simtest.basic.config)"
+    echo "  SIMTEST_AUTO     : define the analysis steps to perform (simtest.auto)"
+    echo "  SIMTEST_THREADS  : set the number of threads for the processing node (0)"
     break
   end
 
-  $testroot = $1
-  $dbname = $2
-  $hostname = $3
-  $threads = $4
+  $dbname = $1
+  $hostname = $2
+  $init = $3
+  if (("$init" != "run") && ("$init" != "inject") && ("$init" != "new"))
+    echo "USAGE: simtest (dbname) (hostname) (init)"
+    echo " if (init) == run    : just run the analysis tasks"
+    echo " if (init) == inject : (re)create the database and (re)inject the images"
+    echo " if (init) == new    : restart completely from scratch"
+    break
+  end    
  
-  if ($5 > 0)
+  if (("$init" == "new") || ("$init" == "new"))  
     # XXX this will fail and exit the script if the db does not exist or is old...
     exec pxadmin -delete -dbname $dbname
@@ -22 +61 @@
     exec pxadmin -create -dbname $dbname
 
-    if ($5 == 2)
+    if ("$init" == "new")
       # the labels "wait" and "proc" are special names used in automate.pro
-      exec ppSimSequence $MODULES:0/$testroot.config $testroot.mkimages $testroot.inject -path raw -workdir work -dbname $dbname -label wait -dvodb DVO -tess_id TESS 
-      # not ready for this yet
-      # -ppsim_recipe FLATCORR
+    
+      $ppsim = "ppSimSequence $MODULES:0/$SIMTEST_SEQUENCE simtest.mkimages simtest.inject -path raw -workdir work -dbname $dbname -label wait -dvodb DVO -tess_id TESS"
+      if ("$PPSIM_RECIPE" != "default") 
+         $ppsim = $ppsim -ppsim_recipe $PPSIM_RECIPE
+      end
+      $ppsim = $ppsim -camera $SIMTEST_CAMERA
 
-      exec source $testroot.mkimages
+      exec $ppsim
 
+      exec source simtest.mkimages
+
+      ## XXX this will need to be optional as well 
       file TESS/Images.dat found
       if ($found == 0)
@@ -37 +82 @@
 
     exec chiptool -dbname $dbname -block -label wait
-    exec source $testroot.inject
+    exec source simtest.inject
   end
 
@@ -45 +90 @@
   module.tasks
 
-  controller host add $hostname
+  if ($SIMTEST_THREADS == 0) 
+    controller host add $hostname
+  else
+    controller host add $hostname -threads $SIMTEST_THREADS
+  end
+
   add.database $dbname
 
-  automate.load simtest.auto $dbname
+  automate.load $SIMTEST_AUTO $SIMTEST_CAMERA $dbname
   run
 end
 
 # XXX add a test function to pxadmin to check for db existence
+
+## currently defined options for sequence, etc:
+
+# sequence: simtest.basic.config, simtest.flatcorr.config
+# auto: simtest.basic.auto, simtest.flatcorr.auto
+
+macro simtest.setup.flatcorr
+  $PPSIM_RECIPE = FLATCORR
+  $SIMTEST_SEQUENCE = simtest.flatcorr.config
+  $SIMTEST_AUTO = simtest.flatcorr.auto
+end