Changeset 29837

Timestamp:

Nov 26, 2010, 10:34:11 AM (15 years ago)

Author:

eugene

Message:

merge updates from trunk

Location:

branches/eam_branches/ipp-20101103/ippTasks

Files:

• . (modified) (1 prop)
• pstamp.pro (modified) (1 diff)
• publish.pro (modified) (4 diffs)
• science.cleanup.pro (modified) (2 diffs)
• stack.pro (modified) (6 diffs)
• survey.pro (modified) (13 diffs)

branches/eam_branches/ipp-20101103/ippTasks/pstamp.pro

@@ -134 +134 @@
     end
     task pstamp.cleanup.run
+        active false
+    end
+end
+macro pstamp.find.on
+    task pstamp.request.find
+        active true
+    end
+end
+macro pstamp.find.off
+    task pstamp.request.find.load
         active false
     end

branches/eam_branches/ipp-20101103/ippTasks/publish.pro

@@ -15 +15 @@
 
 macro publish.on
-  task publish.trigger
-    active true
-  end
   task publish.load
     active true
@@ -30 +27 @@
 
 macro publish.off
-  task publish.trigger
-    active false
-  end
   task publish.load
     active false
@@ -59 +53 @@
 
 # this variable will cycle through the known database names
-$publish_trigger_DB = 0
 $publish_load_DB = 0
 $publish_revert_DB = 0
 
-task           publish.trigger
+task           publish.load
   host         local
 
@@ -72 +65 @@
 
   stdout NULL
-  stderr $LOGDIR/publish.trigger.log
-
-  task.exec
-    if ($LABEL:n == 0) break
-    #otherwise everything will be queued when there is no label.
-    $run = pubtool -definerun
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$publish_trigger_DB
-      $run = $run -dbname $DB:$publish_trigger_DB
-      $publish_trigger_DB ++
-      if ($publish_trigger_DB >= $DB:n) set publish_trigger_DB = 0
-    end
-    add_poll_labels run
-    command $run
-  end
-
-  # success
-  task.exit    0
-  end
-
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-
-  task.exit    crash
-    showcommand crash
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-
-task           publish.load
-  host         local
-
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-
-  stdout NULL
   stderr $LOGDIR/publish.load.log
 
-  # this query should probably be label-based...
   task.exec
     $run = pubtool -pending

branches/eam_branches/ipp-20101103/ippTasks/science.cleanup.pro

@@ -8 +8 @@
     diff.cleanup.on
     stack.cleanup.on
+    chip.bg.cleanup.on
+    warp.bg.cleanup.on
 end
 
@@ -934 +936 @@
 
 
+book init chipBGPendingCleanup
+
+macro chip.bg.cleanup.status
+  book listbook chipBGPendingCleanup
+end
+
+macro chip.bg.cleanup.reset
+  book init chipBGPendingCleanup
+end
+
+macro chip.bg.cleanup.on
+  task chip.bg.cleanup.load
+    active true
+  end
+  task chip.bg.cleanup.run
+    active true
+  end
+end
+
+macro chip.bg.cleanup.off
+  task chip.bg.cleanup.load
+    active false
+  end
+  task chip.bg.cleanup.run
+    active false
+  end
+end
+
+# this variable will cycle through the known database names
+$chip_bg_cleanup_DB = 0
+
+task           chip.bg.cleanup.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+  active       false
+
+  stdout NULL
+  stderr $LOGDIR/chip.bg.cleanup.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = bgtool -pendingcleanupchiprun
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$chip_bg_cleanup_DB
+      $run = $run -dbname $DB:$chip_bg_cleanup_DB
+      $chip_bg_cleanup_DB ++
+      if ($chip_bg_cleanup_DB >= $DB:n) set chip_bg_cleanup_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout chipBGPendingCleanup -key chip_bg_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook chipBGPendingCleanup
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup chipBGPendingCleanup
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# run the ipp_cleanup.pl script on pending images
+task           chip.bg.cleanup.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 300
+  active       false
+
+  task.exec
+    book npages chipBGPendingCleanup -var N
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
+    if ($NETWORK == 0) break
+    
+    # look for new images in chipBGPendingCleanup (pantaskState == INIT)
+    book getpage chipBGPendingCleanup 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword chipBGPendingCleanup $pageName pantaskState RUN
+    book getword chipBGPendingCleanup $pageName camera -var CAMERA
+    book getword chipBGPendingCleanup $pageName state -var CLEANUP_MODE
+    book getword chipBGPendingCleanup $pageName chip_bg_id -var CHIP_BG_ID
+    book getword chipBGPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
+    book getword chipBGPendingCleanup $pageName exp_tag -var EXP_TAG
+    book getword chipBGPendingCleanup $pageName dbname -var DBNAME
+
+    # specify choice of local or remote host based on camera and chip (class_id)
+    set.host.for.camera $CAMERA FPA
+    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
+    sprintf logfile "%s/%s/%s.bgc.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_BG_ID
+
+    stdout $LOGDIR/chip.bg.cleanup.log
+    stderr $LOGDIR/chip.bg.cleanup.log
+
+    # XXX is everything listed here needed?
+    $run = ipp_cleanup.pl --stage chip_bg --stage_id $CHIP_BG_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $logfile
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit chipBGPendingCleanup $options:0 $JOB_STATUS
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword chipBGPendingCleanup $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword chipBGPendingCleanup $options:0 pantaskState TIMEOUT
+  end
+end
+
+book init warpBGPendingCleanup
+
+macro warp.bg.cleanup.status
+  book listbook warpBGPendingCleanup
+end
+
+macro warp.bg.cleanup.reset
+  book init warpBGPendingCleanup
+end
+
+macro warp.bg.cleanup.on
+  task warp.bg.cleanup.load
+    active true
+  end
+  task warp.bg.cleanup.run
+    active true
+  end
+end
+
+macro warp.bg.cleanup.off
+  task warp.bg.cleanup.load
+    active false
+  end
+  task warp.bg.cleanup.run
+    active false
+  end
+end
+
+# this variable will cycle through the known database names
+$warp_bg_cleanup_DB = 0
+
+task           warp.bg.cleanup.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 300
+  npending     1
+  active       false
+
+  stdout NULL
+  stderr $LOGDIR/warp.bg.cleanup.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = bgtool -pendingcleanupwarprun
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$warp_bg_cleanup_DB
+      $run = $run -dbname $DB:$warp_bg_cleanup_DB
+      $warp_bg_cleanup_DB ++
+      if ($warp_bg_cleanup_DB >= $DB:n) set warp_bg_cleanup_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout warpBGPendingCleanup -key warp_bg_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook warpBGPendingCleanup
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup warpBGPendingCleanup
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# run the ipp_cleanup.pl script on pending images
+task           warp.bg.cleanup.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 300
+  active       false
+
+  task.exec
+    book npages warpBGPendingCleanup -var N
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
+    if ($NETWORK == 0) break
+    
+    # look for new images in warpBGPendingCleanup (pantaskState == INIT)
+    book getpage warpBGPendingCleanup 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword warpBGPendingCleanup $pageName pantaskState RUN
+    book getword warpBGPendingCleanup $pageName camera -var CAMERA
+    book getword warpBGPendingCleanup $pageName state -var CLEANUP_MODE
+    book getword warpBGPendingCleanup $pageName warp_bg_id -var WARP_BG_ID
+    book getword warpBGPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
+    book getword warpBGPendingCleanup $pageName exp_tag -var EXP_TAG
+    book getword warpBGPendingCleanup $pageName dbname -var DBNAME
+
+    # specify choice of local or remote host based on camera and warp 
+    set.host.for.camera $CAMERA FPA
+    set.workdir.by.camera $CAMERA $WARP_BG_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+    sprintf logfile "%s/%s/%s.bgw.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_BG_ID
+
+    stdout $LOGDIR/warp.bg.cleanup.log
+    stderr $LOGDIR/warp.bg.cleanup.log
+
+    # XXX is everything listed here needed?
+    $run = ipp_cleanup.pl --stage warp_bg --stage_id $WARP_BG_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $logfile
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit warpBGPendingCleanup $options:0 $JOB_STATUS
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword warpBGPendingCleanup $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword warpBGPendingCleanup $options:0 pantaskState TIMEOUT
+  end
+end

branches/eam_branches/ipp-20101103/ippTasks/stack.pro

@@ -8 +8 @@
 check.globals
 
+if ($?POLL_LIMIT_STACK == 0) set POLL_LIMIT_STACK = 40
+
+macro set.stack.poll
+  if ($0 != 2)
+    echo "USAGE:set.stack.poll (value)"
+    break
+  end
+
+  $POLL_LIMIT_STACK = $1
+end
+macro get.stack.poll
+  echo $POLL_LIMIT_STACK
+end
+
 ### Initialise the books containing the tasks to do
 book init stackSumSkyfile
 book init stackPendingSummary
-#book init stackCleanup
 
 ### Database lists
 $stackSkycell_DB = 0
 $stack_revert_DB = 0
-#$stackCleanup_DB = 0
 $stackSummary_DB = 0
 
@@ -22 +34 @@
 macro stack.status
   book listbook stackSumSkyfile
-#  book listbook stackCleanup
 end
 
@@ -29 +40 @@
   book init stackSumSkyfile
   book init stackPendingSummary
-#  book init stackCleanup
 end
 
@@ -81 +91 @@
   end
 end
-
-
-
-
-
-# macro stack.cleanup.on
-#   task stack.cleanup.load
-#     active true
-#   end
-#   task stack.cleanup.run
-#     active true
-#   end
-# end
-
-# macro stack.cleanup.off
-#   task stack.cleanup.load
-#     active false
-#   end
-#   task stack.cleanup.run
-#     active false
-#   end
-# end
 
 
@@ -132 +120 @@
     add_poll_args run
     add_poll_labels run
-    # reduce the limit (the last one takes precedence)
-    $run = $run -limit 40
+    # change the limit (the last one on the command line takes precedence)
+    $run = $run -limit $POLL_LIMIT_STACK
     command $run
   end
@@ -246 +234 @@
   end
 end
-
-
-# # select images ready for stack analysis
-# # new entries are added to stackCleanup
-# # skip already-present entries
-# task         stack.cleanup.load
-#   host         local
-
-#   periods      -poll $LOADPOLL
-#   periods      -exec $LOADEXEC
-#   periods      -timeout 30
-#   npending     1
-#   active       false
-
-#   stdout NULL
-#   stderr $LOGDIR/stack.cleanup.log
-
-#   task.exec
-#     if ($LABEL:n == 0) break
-#     $run = stacktool -pendingcleanuprun
-#     if ($DB:n == 0)
-#       option DEFAULT
-#     else
-#       # save the DB name for the exit tasks
-#       option $DB:$stackCleanup_DB
-#       $run = $run -dbname $DB:$stackCleanup_DB
-#       $stackCleanup_DB ++
-#       if ($stackCleanup_DB >= $DB:n) set stackCleanup_DB = 0
-#     end
-#     add_poll_args run
-#     add_poll_labels run
-#     command $run
-#   end
-
-#   # success
-#   task.exit    0
-#     # convert 'stdout' to book format
-#     ipptool2book stdout stackCleanup -key stack_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-#     if ($VERBOSE > 2)
-#       book listbook stackCleanup
-#     end
-
-#     # delete existing entries in the appropriate pantaskStates
-#     process_cleanup stackCleanup
-#   end
-
-#   # locked list
-#   task.exit    default
-#     showcommand failure
-#   end
-
-#   task.exit    crash
-#     showcommand crash
-#   end
-
-#   # operation times out?
-#   task.exit    timeout
-#     showcommand timeout
-#   end
-# end
-
-# # run the ipp_cleanup.pl script on pending images
-# task         stack.cleanup.run
-#   periods      -poll $RUNPOLL
-#   periods      -exec $RUNEXEC
-#   periods      -timeout 60
-#   active       false
-
-#   task.exec
-#     book npages stackCleanup -var N
-#     if ($N == 0) break
-#     if ($NETWORK == 0) break
-    
-#     # look for new images in stackCleanup (pantaskState == INIT)
-#     book getpage stackCleanup 0 -var pageName -key pantaskState INIT
-#     if ("$pageName" == "NULL") break
-
-#     book setword stackCleanup $pageName pantaskState RUN
-#     book getword stackCleanup $pageName camera -var CAMERA
-#     book getword stackCleanup $pageName state -var CLEANUP_MODE
-#     book getword stackCleanup $pageName stack_id -var STACK_ID
-#     book getword stackCleanup $pageName dbname -var DBNAME
-
-#     # specify choice of local or remote host based on camera and stack (class_id)
-#     set.host.for.camera $CAMERA FPA
-
-#     stdout $LOGDIR/stack.cleanup.log
-#     stderr $LOGDIR/stack.cleanup.log
-
-#     # XXX is everything listed here needed?
-#     $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE
-#     add_standard_args run
-
-#     # save the pageName for future reference below
-#     options $pageName
-
-#     # create the command line
-#     if ($VERBOSE > 1)
-#       echo command $run
-#     end
-#     command $run
-#   end
-
-#   # default exit status
-#   task.exit    default
-#     process_exit stackCleanup $options:0 $JOB_STATUS
-#   end
-
-#   task.exit    crash
-#     showcommand crash
-#     book setword stackCleanup $options:0 pantaskState CRASH
-#   end
-
-#   # operation timed out?
-#   task.exit    timeout
-#     showcommand timeout
-#     book setword stackCleanup $options:0 pantaskState TIMEOUT
-#   end
-# end
-
-
 
 

branches/eam_branches/ipp-20101103/ippTasks/survey.pro

@@ -32 +32 @@
  book create SURVEY_CHIP_BG
  book create SURVEY_WARP_BG
+ book create SURVEY_PUBLISH
  $haveSurveyBooks = TRUE
 end
@@ -44 +45 @@
 $SURVEY_CHIP_BG_DB = 0
 $SURVEY_WARP_BG_DB = 0
+$SURVEY_PUBLISH_DB = 0
 
 $SURVEY_EXEC = 120
@@ -77 +79 @@
     active true
   end
+  task survey.publish
+    active true
+  end
 end
 
@@ -105 +110 @@
   end
   task survey.warp.bg
+    active false
+  end
+  task survey.publish
     active false
   end
@@ -355 +363 @@
   book listbook SURVEY_WARP_BG
 end
+
+# user functions to manipulate publish labels
+macro survey.add.publish
+  if ($0 != 3)
+    echo "USAGE: survey.add.publish (label) (client_id)"
+    break
+  end
+  book newpage SURVEY_PUBLISH $1
+  book setword SURVEY_PUBLISH $1 CLIENT_ID $2
+  book setword SURVEY_PUBLISH $1 STATE PENDING
+end
+
+macro survey.del.publish
+  if ($0 != 2)
+    echo "USAGE: survey.del.publish (label)"
+    break
+  end
+  book delpage SURVEY_PUBLISH $1
+end
+
+macro survey.show.publish
+  if ($0 != 1)
+    echo "USAGE: survey.show.publish"
+    break
+  end
+  book listbook SURVEY_PUBLISH
+end
+
 
 task survey.diff
@@ -579 +615 @@
     end
     
-    echo $run
+    # echo $run
     command $run
   end
@@ -651 +687 @@
     end
     
-    echo $run
+    # echo $run
     command $run
   end
@@ -674 +710 @@
   end
 end
+
 
 task survey.addstar
@@ -723 +760 @@
     
     $run = $run -definebyquery -destreaked -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group -set_minidvodb -set_label $minidvodb_group
-    echo $run
+    # echo $run
     command $run
   end
@@ -790 +827 @@
       option DEFAULT
     else
-      $run = $run --dbname $DB:$SURVEY_MAGIC_DB
-      option $DB:$SURVEY_MAGIC_DB
+      $run = $run --dbname $DB:$SURVEY_DESTREAK_DB
+      option $DB:$SURVEY_DESTREAK_DB
     end
     
@@ -934 +971 @@
     end
     
-    echo $run
+    # echo $run
     command $run
   end
@@ -1004 +1041 @@
     end
     
-    echo $run
+    # echo $run
     command $run
   end
@@ -1027 +1064 @@
   end
 end
+
+task survey.publish
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.publish.log
+  stderr $LOGDIR/survey.publish.log
+
+  # generate publish runs
+  task.exec
+    book npages SURVEY_PUBLISH -var N
+    if ($N == 0)
+    #  echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_PUBLISH 0 -var label -key STATE NEW
+    if ("$label" == "NULL")
+      # All labels have been done --- reset
+      # echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_PUBLISH $i -var label
+        book setword SURVEY_PUBLISH $label STATE NEW
+      end
+      book getpage SURVEY_PUBLISH 0 -var label -key STATE NEW
+
+      # Select different database
+      $SURVEY_PUBLISH ++
+      if ($SURVEY_PUBLISH >= $DB:n) set SURVEY_PUBLISH = 0
+    end
+
+    book setword SURVEY_PUBLISH $label STATE DONE
+    book getword SURVEY_PUBLISH $label CLIENT_ID -var client_id
+  
+    $run = pubtool -definerun -label $label -client_id $client_id
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run -dbname $DB:$SURVEY_PUBLISH
+      option $DB:$SURVEY_PUBLISH
+    end
+    
+    #echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    #echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end