Changeset 34775

Timestamp:

Dec 7, 2012, 11:56:15 AM (13 years ago)

Author:

bills

Message:

For chip, warp, and diff stage cleanup, add a mode where all components
are cleaned regardless of their data state. This can be used to go
and clean up files added to the list of files to clean (chip and warp cmfs)
for example and to find files that weren't previously cleaned up.
Controlled with set.check.all.components and clear.check.all.components.

Log output to nebulous logfile rather than a file in pantasks_logs

File:

• trunk/ippTasks/science.cleanup.pro (modified) (8 diffs)

trunk/ippTasks/science.cleanup.pro

@@ -13 +13 @@
 
 check.globals
+
+# if set tell cleanup script to check for files from all components regardless of the data_state
+$CHECK_ALL_COMPONENTS = 0
+macro set.check.all.components
+    $CHECK_ALL_COMPONENTS = 1
+end
+macro clear.check.all.components
+    $CHECK_ALL_COMPONENTS = 0
+end
+macro get.check.all.components
+    echo CHECK_ALL_COMPONENTS = $CHECK_ALL_COMPONENTS
+end
+
 
 ## chip.cleanup.pro : -*- sh -*-
@@ -130 +143 @@
     book getword chipPendingCleanup $pageName state -var CLEANUP_MODE
     book getword chipPendingCleanup $pageName chip_id -var CHIP_ID
+    book getword chipPendingCleanup $pageName exp_tag -var EXP_TAG
+    book getword chipPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
     book getword chipPendingCleanup $pageName dbname -var DBNAME
 
-    # specify choice of local or remote host based on camera and chip (class_id)
+    # specify choice of local or remote host based on camera and chip (in this case FPA)
     set.host.for.camera $CAMERA FPA
+
+    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    sprintf LOGFILE "%s/%s/%s.ch.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID
 
     stdout $LOGDIR/chip.cleanup.log
     stderr $LOGDIR/chip.cleanup.log
 
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
+    if ($CHECK_ALL_COMPONENTS)
+        $run = $run --check-all
+    end
     add_standard_args run
 
@@ -282 +303 @@
     book getword camPendingCleanup $pageName state  -var CLEANUP_MODE
     book getword camPendingCleanup $pageName cam_id -var CAM_ID
+    book getword camPendingCleanup $pageName exp_tag -var EXP_TAG
+    book getword camPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
     book getword camPendingCleanup $pageName dbname -var DBNAME
 
@@ -287 +310 @@
     set.host.for.camera $CAMERA FPA
 
+    # set the WORKDIR variable
+    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    ## generate LOGFILE specific to this exposure (& cam_id)
+    sprintf LOGFILE "%s/%s/%s.cm.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID
+
     stdout $LOGDIR/camera.cleanup.log
     stderr $LOGDIR/camera.cleanup.log
 
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
     add_standard_args run
 
@@ -591 +619 @@
     book getword warpPendingCleanup $pageName state -var CLEANUP_MODE
     book getword warpPendingCleanup $pageName warp_id -var WARP_ID
+    book getword warpPendingCleanup $pageName exp_tag -var EXP_TAG
+    book getword warpPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
     book getword warpPendingCleanup $pageName dbname -var DBNAME
 
     # specify choice of local or remote host based on camera and warp (class_id)
     set.host.for.camera $CAMERA FPA
+    set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    sprintf LOGFILE "%s/%s/%s.wrp.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID
 
     stdout $LOGDIR/warp.cleanup.log
@@ -600 +633 @@
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
+    if ($CHECK_ALL_COMPONENTS)
+        $run = $run --check-all
+    end
     add_standard_args run
 
@@ -748 +784 @@
     book getword diffCleanup $pageName state -var CLEANUP_MODE
     book getword diffCleanup $pageName diff_id -var DIFF_ID
+    book getword diffCleanup $pageName tess_id -var TESS_ID
+    book getword diffCleanup $pageName workdir -var WORKDIR_TEMPLATE
     book getword diffCleanup $pageName dbname -var DBNAME
 
     # specify choice of local or remote host based on camera and diff (class_id)
     set.host.for.camera $CAMERA FPA
+    set.workdir.by.camera $CAMERA $DIFF_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+    sprintf LOGFILE "%s/%s/%s.dif.%s.cleanup.log" $WORKDIR $TESS_ID $TESS_ID $DIFF_ID
+
 
     stdout $LOGDIR/diff.cleanup.log
@@ -757 +798 @@
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
+    if ($CHECK_ALL_COMPONENTS)
+        $run = $run --check-all
+    end
     add_standard_args run