Changeset 9104 for trunk/ippTasks/detrend.process.pro

Timestamp:

Oct 2, 2006, 12:36:59 PM (20 years ago)

Author:

eugene

Message:

updates to match new refactoring of tables

File:

• trunk/ippTasks/detrend.process.pro (modified) (5 diffs)

trunk/ippTasks/detrend.process.pro

@@ -9 +9 @@
 # detrend_process.pl 1 850131b mc bias 850131b.fits 850131b
 
-## these tasks use the queue DetrendImfilesToProcess
-## the DetrendImfilesToProcess queue contains:
+## these tasks use the queue DetrendProcessImfiles
+## the DetrendProcessImfiles queue contains:
 ## STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI 
 
-queueinit DetrendImfilesToProcess
+queueinit DetrendProcessImfiles
+queueinit DetrendProcessExposures
 if ($?network == 0)
   $network = 1
@@ -22 +23 @@
 
 # select images ready for copy 
-# new entries are added to queue DetrendImfilesToProcess
+# new entries are added to queue DetrendProcessImfiles
 # compare the new list with the ones already selected
-task           dettool.raw.load
+task           dettool.process.load
   command      dettool -raw -simple
   host         local
@@ -53 +54 @@
       $CLASS_ID = $word:4
       $URI      = $word:5
-      queuepush DetrendImfilesToProcess -uniq -key 1:3:5 "NEW $DET_ID $TYPE $EXP_KEY $CLASS $CLASS_ID $URI"
+      queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $TYPE $EXP_KEY $CLASS $CLASS_ID $URI"
     end
 
     # delete existing entries which are DONE
-    queuedrop DetrendImfilesToProcess -key 0 DONE
-  end
-
-  # locked list
-  task.exit    1
-    echo       "new.images: exec failure"
-  end
-
-  # default exit status
-  task.exit    default
-    echo       "new.images: unknown exit status: $EXIT"
-  end
-
-  # operation times out?
-  task.exit    timeout
-    echo       "new.images: timeout"
+    queuedrop DetrendProcessImfiles -key 0 DONE
+  end
+
+  # error
+  task.exit    default
+    echo       "detrend.process: failure"
+  end
+
+  # operation times out?
+  task.exit    timeout
+    echo       "detrend.process: timeout"
   end
 end
 
 # copy new images, sending job to desired host
-task           dettool.raw.process
+task           dettool.process.run
   periods      -poll 0.5
   periods      -exec 2.0
@@ -83 +79 @@
 
   task.exec
-    queuesize DetrendImfilesToProcess -var N
+    queuesize DetrendProcessImfiles -var N
     if ($N == 0) break
     if ($network == 0) break
     
-    # look for new images on the internal DetrendImfilesToProcess queue
+    # look for new images on the internal DetrendProcessImfiles queue
     # caution with these 'if' statements: syntax errors 
     # will make the task fail without given a good status
-    queuepop DetrendImfilesToProcess -var line -key 0 NEW
+    queuepop DetrendProcessImfiles -var line -key 0 NEW
     if ("$line" == "NULL") break
 
     strpop line state
-    queuepush DetrendImfilesToProcess -replace -key 1:3:5 "RUN $line"
-
-    ## the DetrendImfilesToProcess queue contains:
+    queuepush DetrendProcessImfiles -replace -key 1:3:5 "RUN $line"
+
+    ## the DetrendProcessImfiles queue contains:
     # STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI 
     ## NOTE: the queue has the STATE prepended 
@@ -119 +115 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_process.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI
-    command detrend_process.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI
+    echo command detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI
+    command detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI
   end
 
   # success
   task.exit 0
-    queuepush DetrendImfilesToProcess -replace -key 1:3:5 "DONE $options:0"
+    queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0"
   end
 
   # default exit status
   task.exit    default
-    queuepush DetrendImfilesToProcess -replace -key 1:3:5 "FAIL $options:0"
-  end
-
-  # operation times out?
-  task.exit    timeout
-    queuepush DetrendImfilesToProcess -replace -key 1:3:5 "TIMEOUT $options:0"
-  end
-end
+    queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0"
+  end
+
+  # operation times out?
+  task.exit    timeout
+    queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0"
+  end
+end
+
+# select images ready for copy 
+# new entries are added to queue DetrendProcessExposures
+# compare the new list with the ones already selected
+task           dettool.processexp.load
+  command      dettool -toprocessdexp -simple
+  host         local
+
+  periods      -poll 1
+  periods      -exec 5
+  periods      -timeout 30
+
+  stdout dettool.processexp.log
+  stderr dettool.processexp.log
+
+  # success
+  task.exit    0
+    # XXX is it necessary for these to be local?
+    # drop after this task macro is done?
+    local i Nqueue
+
+    # compare output with newImage queue
+    # only add entries which don't exist in queue
+    queuesize stdout -var Nqueue
+    for i 0 $Nqueue
+      queuepop stdout -var line
+      list word -split $line
+      $DET_ID   = $word:0
+      $TYPE     = $word:1
+      $EXP_KEY  = $word:2
+      queuepush DetrendProcessExposures -uniq -key 1:3 "NEW $DET_ID $TYPE $EXP_KEY"
+    end
+
+    # delete existing entries which are DONE
+    queuedrop DetrendProcessExposures -key 0 DONE
+  end
+
+  # error
+  task.exit    default
+    echo       "detrend.process: failure"
+  end
+
+  # operation times out?
+  task.exit    timeout
+    echo       "detrend.process: timeout"
+  end
+end
+
+# copy new images, sending job to desired host
+task           dettool.processexp.run
+  periods      -poll 0.5
+  periods      -exec 2.0
+  periods      -timeout 60
+
+  task.exec
+    queuesize DetrendProcessExposures -var N
+    if ($N == 0) break
+    if ($network == 0) break
+    
+    # look for new images on the internal DetrendProcessExposures queue
+    # caution with these 'if' statements: syntax errors 
+    # will make the task fail without given a good status
+    queuepop DetrendProcessExposures -var line -key 0 NEW
+    if ("$line" == "NULL") break
+
+    strpop line state
+    queuepush DetrendProcessExposures -replace -key 1:3:5 "RUN $line"
+
+    ## the DetrendProcessExposures queue contains:
+    # STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI 
+    ## NOTE: the queue has the STATE prepended 
+    list word -split $line
+    $DET_ID   = $word:0
+    $TYPE     = $word:1
+    $EXP_KEY  = $word:2
+
+    stdout $EXP_KEY.detproc.$CLASS_ID.log
+    stderr $EXP_KEY.detproc.$CLASS_ID.log
+
+    # specify choice of remote host:
+    if ($parallel)
+      host anyhost
+    else
+      host local
+    end
+
+    # create example job options as a demonstration
+    options "$line"
+    echo command detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_KEY --det_type bias
+    command detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_KEY --det_type bias
+  end
+
+  # success
+  task.exit 0
+    queuepush DetrendProcessExposures -replace -key 1:3:5 "DONE $options:0"
+  end
+
+  # default exit status
+  task.exit    default
+    queuepush DetrendProcessExposures -replace -key 1:3:5 "FAIL $options:0"
+  end
+
+  # operation times out?
+  task.exit    timeout
+    queuepush DetrendProcessExposures -replace -key 1:3:5 "TIMEOUT $options:0"
+  end
+end