Changeset 31863 for trunk/ippToPsps/jython/ipptopsps.py

Timestamp:

Jul 12, 2011, 8:35:57 AM (15 years ago)

Author:

rhenders

Message:

restructured to that it polls indefinitely looking for new P2 then ST items; batch-type command-line arg is now optional; added a lot of comments

File:

• trunk/ippToPsps/jython/ipptopsps.py (modified) (2 diffs)

trunk/ippToPsps/jython/ipptopsps.py

@@ -14 +14 @@
 from detectionbatch import DetectionBatch
 
-# check we have a batch type
-if len(sys.argv) < 2:
-    print "\nERROR: please provie a batchType: P2 or ST\n"
-    sys.exit(1)
+'''
+Queues up all available stuff for this batch type
+'''
+def queueItems(batchType):
 
-batchType  = sys.argv[1]
+    logger.infoSeparator()
 
-logging.config.fileConfig("logging.conf")
-logging.setLoggerClass(PSLogger)
-logger = logging.getLogger("ippToPspsLog")
-logger.setLevel(logging.INFO)
+    # gett list of all items available in DVO
+    allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType)
+    logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs))
 
-configDoc = ElementTree(file="config.xml")
-dvoGpc1Label = configDoc.find("dvo/gpc1Label").text
+    # if in FORCE mode, then queue full list
+    if FORCE: 
+        ids = allIDs
 
-gpc1Db = Gpc1Db(logger, configDoc)
-ippToPspsDb = IppToPspsDb(logger, configDoc)
-POLLPERIOD = 600
-FORCE = int(configDoc.find("options/force").text)
-TEST = int(configDoc.find("options/testMode").text)
-EPOCH = configDoc.find("options/epoch").text
-PUBLISH = int(configDoc.find("options/publishToDatastore").text)
+    # if not in TEST mode, then only queue un-processed items
+    else:
+        processedIDs = ippToPspsDb.getProcessedIDsForThisStage(batchType, EPOCH)
+        logger.infoPair("Processed %s items" % batchType, "%d" % len(processedIDs))
+        ids = list(set(allIDs) - set(processedIDs))
+        logger.infoPair("Unprocessed %s items" % batchType, "%d" % len(ids))
 
-if FORCE and PUBLISH:
-   response = raw_input("Are you sure you want to publish data with the 'force' option enabled (y/n)? ")
-   if response != "y": sys.exit(1)
+    logger.infoPair("%s items queued" % batchType, "%d" % len(ids))
 
-logger.infoSeparator()
-logger.infoPair("Batch type", batchType)
-logger.infoPair("Loading epoch", EPOCH)
-logger.infoPair("Forcing?", "%d" % FORCE)
-logger.infoPair("Test mode?", "%d" % TEST)
-logger.infoPair("DVO gpc1 label", dvoGpc1Label)
-
-while True:
-
-    if FORCE: 
-        logger.infoPair("Querying GPC1 for", "all available IDs")
-        ids = gpc1Db.getIDsInThisDVODbForThisStage(dvoGpc1Label, batchType)
-
-    else:
-        logger.infoPair("Querying GPC1 for", "all unprocessed IDs")
-        ids = ippToPspsDb.getUnprocessedIDsForThisStage(dvoGpc1Label, batchType, EPOCH)
-
-    logger.infoPair("Queuing", "%d items" % len(ids))
-
+    # loop round IDs of all items to be processed
     for id in ids:
 
@@ -80 +59 @@
         batch.run()
 
+        # if in TEST mode, then quit after one batch
         if TEST: break
 
+
+'''
+Start of program.
+'''
+
+# check to see if we have a batch type
+if len(sys.argv) > 1: BATCHTYPE  = sys.argv[1]
+else: BATCHTYPE = "ALL"
+
+# set up logging
+logging.config.fileConfig("logging.conf")
+logging.setLoggerClass(PSLogger)
+logger = logging.getLogger("ippToPspsLog")
+logger.setLevel(logging.INFO)
+
+# open config file
+configDoc = ElementTree(file="config.xml")
+
+# create database objects
+gpc1Db = Gpc1Db(logger, configDoc)
+ippToPspsDb = IppToPspsDb(logger, configDoc)
+
+# get values from the configutaion file
+POLLPERIOD = 600
+DVOGPC1LABEL = configDoc.find("dvo/gpc1Label").text
+FORCE = int(configDoc.find("options/force").text)
+TEST = int(configDoc.find("options/testMode").text)
+EPOCH = configDoc.find("options/epoch").text
+PUBLISH = int(configDoc.find("options/publishToDatastore").text)
+
+# prompt user: FORCE and PUBLISH is a dangerous combination
+if FORCE and PUBLISH:
+   response = raw_input("Are you sure you want to publish data with the 'force' option enabled (y/n)? ")
+   if response != "y": sys.exit(1)
+
+# print info to log
+logger.infoSeparator()
+logger.infoPair("Batch type", BATCHTYPE)
+logger.infoPair("Loading epoch", EPOCH)
+logger.infoPair("Forcing?", "%d" % FORCE)
+logger.infoPair("Test mode?", "%d" % TEST)
+logger.infoPair("DVO gpc1 label", DVOGPC1LABEL)
+
+'''
+Main processing loop:
+
+ 1) queues available P2 batches
+ 2) queues available ST batches
+ 3) if in test mode, quits
+ 4) otherwise, waits then repeats from 1
+
+'''
+while True:
+
+    # queue all items for all batch types, sequentially
+    if BATCHTYPE == "ALL":
+        queueItems("P2")
+        queueItems("ST")
+    # queue only items for the user-specified batch type
+    else:
+        queueItems(BATCHTYPE)
+
+    # in the TEST mode, we quit after submitting one batch
     if TEST: break
 
-    # wait for ten minutes before checking for new ids
+    # wait for the POLLPERIOD before checking for new ids
     logger.infoPair("Finished. Waiting for", "%.1f minutes before checking DVO for new items" % (POLLPERIOD/60.0))
     time.sleep(POLLPERIOD)
 
 
-