Changeset 19090

Timestamp:

Aug 16, 2008, 1:07:21 PM (18 years ago)

Author:

eugene

Message:

adding cleanup stages

Location:

trunk/ippTasks

Files:

• camera.pro (modified) (4 diffs)
• detrend.process.pro (modified) (10 diffs)
• fake.pro (modified) (4 diffs)
• warp.pro (modified) (6 diffs)

trunk/ippTasks/camera.pro

@@ -7 +7 @@
 
 book init camPendingExp
+book init camPendingCleanup
 
 macro camera.status
   book listbook camPendingExp
+  book listbook camPendingCleanup
 end
 
 macro camera.reset
   book init camPendingExp
+  book init camPendingCleanup
 end
 
@@ -23 +26 @@
     active true
   end
+  task camera.cleanup.load
+    active false
+  end
+  task camera.cleanup.run
+    active false
+  end
 end
 
@@ -30 +39 @@
   end
   task camera.exp.run
+    active false
+  end
+  task camera.cleanup.load
+    active false
+  end
+  task camera.cleanup.run
     active false
   end
@@ -164 +179 @@
 end
 
-## XXX add a global path to output files  
+# this variable will cycle through the known database names
+$camera_cleanup_DB = 0
+
+# select images ready for cam analysis
+# new entries are added to camPendingImfile
+# skip already-present entries
+task           camera.cleanup.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+  active       true
+
+  stdout NULL
+  stderr $LOGDIR/camera.cleanup.log
+
+  task.exec
+    if ($DB:n == 0)
+      option DEFAULT
+      command camtool -pendingcleanuprun -limit 20
+    else
+      # save the DB name for the exit tasks
+      option $DB:$camera_cleanup_DB
+      command camtool -pendingcleanuprun -limit 20 -dbname $DB:$camera_cleanup_DB
+      $camera_cleanup_DB ++
+      if ($camera_cleanup_DB >= $DB:n) set camera_cleanup_DB = 0
+    end
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout camPendingCleanup -key cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook camPendingCleanup
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup camPendingCleanup
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# run the ipp_cleanup.pl script on pending images
+task           camera.cleanup.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+  active       true
+
+  task.exec
+    book npages camPendingCleanup -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images in camPendingCleanup (pantaskState == INIT)
+    book getpage camPendingCleanup 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword camPendingCleanup $pageName pantaskState RUN
+    book getword camPendingCleanup $pageName camera -var CAMERA
+    book getword camPendingCleanup $pageName state  -var CLEANUP_MODE
+    book getword camPendingCleanup $pageName cam_id -var CAM_ID
+    book getword camPendingCleanup $pageName dbname -var DBNAME
+
+    # specify choice of local or remote host based on camera and cam (class_id)
+    set.host.for.camera $CAMERA FPA
+
+    stdout $LOGDIR/camera.cleanup.log
+    stderr $LOGDIR/camera.cleanup.log
+
+    # XXX is everything listed here needed?
+    $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit camPendingCleanup $options:0 $JOB_STATUS
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword camPendingCleanup $options:0 pantaskState TIMEOUT
+  end
+end

trunk/ippTasks/detrend.process.pro

@@ -8 +8 @@
 book init detPendingProcessedImfile
 book init detPendingProcessedExp
+book init detCleanupProcessedImfile
+book init detCleanupProcessedExp
 
 macro detproc.reset
   book init detPendingProcessedImfile
   book init detPendingProcessedExp
+  book init detCleanupProcessedImfile
+  book init detCleanupProcessedExp
 end
 
@@ -19 +23 @@
   echo detPendingProcessedExp
   book listbook detPendingProcessedExp
+  echo detCleanupProcessedImfile
+  book listbook detCleanupProcessedImfile
+  echo detCleanupProcessedExp
+  book listbook detCleanupProcessedExp
 end
 
@@ -34 +42 @@
     active true
   end
+  task detrend.cleanup.process.load
+    active true
+  end
+  task detrend.cleanup.process.run
+    active true
+  end
+  task detrend.cleanup.processexp.load
+    active true
+  end
+  task detrend.cleanup.processexp.run
+    active true
+  end
 end
 
@@ -47 +67 @@
   end
   task detrend.processexp.run
+    active false
+  end
+  task detrend.cleanup.process.load
+    active false
+  end
+  task detrend.cleanup.process.run
+    active false
+  end
+  task detrend.cleanup.processexp.load
+    active false
+  end
+  task detrend.cleanup.processexp.run
     active false
   end
@@ -55 +87 @@
 $detPendingProcessedImfile_DB = 0
 $detPendingProcessedExp_DB = 0
+$detCleanupProcessedImfile_DB = 0
+$detCleanupProcessedExp_DB = 0
 
 # select images ready for copy 
@@ -68 +102 @@
 
   stdout NULL
-  stderr $LOGDIR/detproc.imfile.load.log
+  stderr $LOGDIR/detrend.process.imfile.log
 
   task.exec
@@ -142 +176 @@
     sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID
 
-    stdout $LOGDIR/detproc.imfile.run.log
-    stderr $LOGDIR/detproc.imfile.run.log
+    stdout $LOGDIR/detrend.process.imfile.log
+    stderr $LOGDIR/detrend.process.imfile.log
 
     $run = detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA --outroot $outroot --redirect-output
@@ -185 +219 @@
 
   stdout NULL
-  stderr $LOGDIR/detproc.exp.load.log
+  stderr $LOGDIR/detrend.process.exp.log
 
   task.exec
@@ -257 +291 @@
     sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID
 
-    stdout $LOGDIR/detproc.exp.run.log
-    stderr $LOGDIR/detproc.exp.run.log
+    stdout $LOGDIR/detrend.process.exp.log
+    stderr $LOGDIR/detrend.process.exp.log
 
     $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose
@@ -287 +321 @@
   end
 end
+
+######## cleanup process imfile ########
+task           detrend.cleanup.process.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+  active       true
+
+  stdout NULL
+  stderr $LOGDIR/detrend.cleanup.process.imfile.log
+
+  task.exec
+    if ($DB:n == 0)
+      option DEFAULT
+      command dettool -pendingcleanup_processedimfile -limit 20
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detCleanupProcessedImfile_DB
+      command dettool -pendingcleanup_processedimfile -limit 20 -dbname $DB:$detCleanupProcessedImfile_DB
+      $detCleanupProcessedImfile_DB ++
+      if ($detCleanupProcessedImfile_DB >= $DB:n) set detCleanupProcessedImfile_DB = 0
+    end
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout detCleanupProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook detCleanupProcessedImfile
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup detCleanupProcessedImfile
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# run the ipp_cleanup.pl script on pending images
+task           detrend.cleanup.process.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+  active       true
+
+  task.exec
+    book npages detCleanupProcessedImfile -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images in detCleanupProcessedImfile (pantaskState == INIT)
+    book getpage detCleanupProcessedImfile 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword detCleanupProcessedImfile $pageName pantaskState RUN
+    book getword detCleanupProcessedImfile $pageName det_id   -var DET_ID   
+    book getword detCleanupProcessedImfile $pageName exp_id   -var EXP_ID   
+    book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID 
+    book getword detCleanupProcessedImfile $pageName camera   -var CAMERA
+    book getword detCleanupProcessedImfile $pageName state    -var CLEANUP_MODE
+    book getword detCleanupProcessedImfile $pageName dbname   -var DBNAME
+
+    # specify choice of local or remote host based on camera and chip (class_id)
+    set.host.for.camera $CAMERA FPA
+
+    stdout $LOGDIR/detrend.cleanup.process.imfile.log
+    stderr $LOGDIR/detrend.cleanup.process.imfile.log
+
+    # XXX is everything listed here needed?
+    $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit detCleanupProcessedImfile $options:0 $JOB_STATUS
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword detCleanupProcessedImfile $options:0 pantaskState TIMEOUT
+  end
+end
+ 
+
+######## cleanup process exp ########
+task           detrend.cleanup.processexp.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+  active       true
+
+  stdout NULL
+  stderr $LOGDIR/detrend.cleanup.process.exp.log
+
+  task.exec
+    if ($DB:n == 0)
+      option DEFAULT
+      command dettool -pendingcleanup_processedexp -limit 20
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detCleanupProcessedExp_DB
+      command dettool -pendingcleanup_processedexp -limit 20 -dbname $DB:$detCleanupProcessedExp_DB
+      $detCleanupProcessedExp_DB ++
+      if ($detCleanupProcessedExp_DB >= $DB:n) set detCleanupProcessedExp_DB = 0
+    end
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout detCleanupProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook detCleanupProcessedExp
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup detCleanupProcessedExp
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# run the ipp_cleanup.pl script on pending images
+task           detrend.cleanup.processexp.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+  active       true
+
+  task.exec
+    book npages detCleanupProcessedExp -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images in detCleanupProcessedExp (pantaskState == INIT)
+    book getpage detCleanupProcessedExp 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword detCleanupProcessedExp $pageName pantaskState RUN
+    book getword detCleanupProcessedExp $pageName det_id   -var DET_ID   
+    book getword detCleanupProcessedExp $pageName exp_id   -var EXP_ID   
+    book getword detCleanupProcessedExp $pageName camera -var CAMERA
+    book getword detCleanupProcessedExp $pageName state -var CLEANUP_MODE
+    book getword detCleanupProcessedExp $pageName dbname -var DBNAME
+
+    # specify choice of local or remote host based on camera and chip (class_id)
+    set.host.for.camera $CAMERA FPA
+
+    stdout $LOGDIR/detrend.cleanup.process.exp.log
+    stderr $LOGDIR/detrend.cleanup.process.exp.log
+
+    # XXX is everything listed here needed?
+    $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --exp_id $EXP_ID --camera $CAMERA --mode $CLEANUP_MODE
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit detCleanupProcessedExp $options:0 $JOB_STATUS
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword detCleanupProcessedExp $options:0 pantaskState TIMEOUT
+  end
+end

trunk/ippTasks/fake.pro

@@ -7 +7 @@
 
 book init fakePendingImfile
+book init fakePendingCleanup
 
 macro fake.status
   book listbook fakePendingImfile
+  book listbook fakePendingCleanup
 end
 
 macro fake.reset
   book init fakePendingImfile
+  book init fakePendingCleanup
 end
 
@@ -23 +26 @@
     active true
   end
+  task fake.cleanup.load
+    active false
+  end
+  task fake.cleanup.run
+    active false
+  end
 end
 
@@ -30 +39 @@
   end
   task fake.imfile.run
+    active false
+  end
+  task fake.cleanup.load
+    active false
+  end
+  task fake.cleanup.run
     active false
   end
@@ -154 +169 @@
 end
 
-
+# this variable will cycle through the known database names
+$fake_cleanup_DB = 0
+
+# select images ready for fake analysis
+# new entries are added to fakePendingImfile
+# skip already-present entries
+task           fake.cleanup.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+  active       true
+
+  stdout NULL
+  stderr $LOGDIR/fake.cleanup.log
+
+  task.exec
+    if ($DB:n == 0)
+      option DEFAULT
+      command faketool -pendingcleanuprun -limit 20
+    else
+      # save the DB name for the exit tasks
+      option $DB:$fake_cleanup_DB
+      command faketool -pendingcleanuprun -limit 20 -dbname $DB:$fake_cleanup_DB
+      $fake_cleanup_DB ++
+      if ($fake_cleanup_DB >= $DB:n) set fake_cleanup_DB = 0
+    end
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout fakePendingCleanup -key fake_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook fakePendingCleanup
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup fakePendingCleanup
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# run the ipp_cleanup.pl script on pending images
+task           fake.cleanup.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+  active       true
+
+  task.exec
+    book npages fakePendingCleanup -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images in fakePendingCleanup (pantaskState == INIT)
+    book getpage fakePendingCleanup 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword fakePendingCleanup $pageName pantaskState RUN
+    book getword fakePendingCleanup $pageName camera -var CAMERA
+    book getword fakePendingCleanup $pageName state -var CLEANUP_MODE
+    book getword fakePendingCleanup $pageName fake_id -var FAKE_ID
+    book getword fakePendingCleanup $pageName dbname -var DBNAME
+
+    # specify choice of local or remote host based on camera and fake (class_id)
+    set.host.for.camera $CAMERA FPA
+
+    stdout $LOGDIR/fake.cleanup.log
+    stderr $LOGDIR/fake.cleanup.log
+
+    # XXX is everything listed here needed?
+    $run = ipp_cleanup.pl --stage fake --stage_id $FAKE_ID --camera $CAMERA --mode $CLEANUP_MODE
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit fakePendingCleanup $options:0 $JOB_STATUS
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword fakePendingCleanup $options:0 pantaskState TIMEOUT
+  end
+end

trunk/ippTasks/warp.pro

@@ -12 +12 @@
 book init warpInputExp
 book init warpPendingSkyCell
+book init warpPendingCleanup
 
 ### Database lists
 $warpExp_DB = 0
 $warpSkycell_DB = 0
+$warpCleanup_DB = 0
 
 ### Check status of warping tasks
@@ -21 +23 @@
   book listbook warpInputExp
   book listbook warpPendingSkyCell
+  book listbook warpPendingCleanup
 end
 
@@ -27 +30 @@
   book init warpInputExp
   book init warpPendingSkyCell
+  book init warpPendingCleanup
 end
 
@@ -43 +47 @@
     active true
   end
+  task warp.cleanup.load
+    active true
+  end
+  task warp.cleanup.run
+    active true
+  end
 end
 
@@ -57 +67 @@
   end
   task warp.skycell.run
+    active false
+  end
+  task warp.cleanup.load
+    active false
+  end
+  task warp.cleanup.run
     active false
   end
@@ -283 +299 @@
 end
 
+# select images ready for warp analysis
+# new entries are added to warpPendingImfile
+# skip already-present entries
+task           warp.cleanup.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+  active       true
+
+  stdout NULL
+  stderr $LOGDIR/warp.cleanup.log
+
+  task.exec
+    if ($DB:n == 0)
+      option DEFAULT
+      command warptool -pendingcleanuprun -limit 20
+    else
+      # save the DB name for the exit tasks
+      option $DB:$warpCleanup_DB
+      command warptool -pendingcleanuprun -limit 20 -dbname $DB:$warpCleanup_DB
+      $warpCleanup_DB ++
+      if ($warpCleanup_DB >= $DB:n) set warpCleanup_DB = 0
+    end
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout warpPendingCleanup -key warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook warpPendingCleanup
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup warpPendingCleanup
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# run the ipp_cleanup.pl script on pending images
+task           warp.cleanup.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+  active       true
+
+  task.exec
+    book npages warpPendingCleanup -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images in warpPendingCleanup (pantaskState == INIT)
+    book getpage warpPendingCleanup 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword warpPendingCleanup $pageName pantaskState RUN
+    book getword warpPendingCleanup $pageName camera -var CAMERA
+    book getword warpPendingCleanup $pageName state -var CLEANUP_MODE
+    book getword warpPendingCleanup $pageName warp_id -var WARP_ID
+    book getword warpPendingCleanup $pageName dbname -var DBNAME
+
+    # specify choice of local or remote host based on camera and warp (class_id)
+    set.host.for.camera $CAMERA FPA
+
+    stdout $LOGDIR/warp.cleanup.log
+    stderr $LOGDIR/warp.cleanup.log
+
+    # XXX is everything listed here needed?
+    $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit warpPendingCleanup $options:0 $JOB_STATUS
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword warpPendingCleanup $options:0 pantaskState TIMEOUT
+  end
+end