- Timestamp:
- Mar 30, 2012, 2:49:37 PM (14 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20111122/ippToPsps/jython/cleanup.py
r32543 r33638 9 9 import logging.config 10 10 11 from pslogger import PSLogger 12 from ipptopspsdb import IppToPspsDb 11 from ipptopsps import IppToPsps 13 12 from batch import Batch 14 13 from datastore import Datastore 15 14 from dxlayer import DXLayer 16 15 17 '''18 Finds stuff to delete for this batch type, then:19 20 - removes from the datastore21 - deletes from local disk22 - deletes files from DXLayer23 16 24 17 ''' 25 def clean(batchType): 18 Cleanup class 19 ''' 20 class Cleanup(IppToPsps): 26 21 27 logger.infoSeparator() 28 logger.infoTitle("Cleaning up batch type %s" % batchType) 22 ''' 23 Constructor 24 ''' 25 def __init__(self, argv): 26 super(Cleanup, self).__init__(argv) 29 27 30 localIDs = ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, EPOCH, DVOLABEL) 31 datastoreIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, EPOCH, DVOLABEL) 32 dxlayerIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, EPOCH, DVOLABEL) 33 logger.infoPair("Merged but still on local disk", "%d" % len(localIDs)) 34 logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(datastoreIDs)) 35 logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(dxlayerIDs)) 28 self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) 29 self.dxlayer = DXLayer(self.logger) 30 if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) 36 31 37 if DELETELOCAL: 38 count = 0 39 for id in localIDs: 32 33 ''' 34 Runs... 35 ''' 36 def run(self): 37 38 if self.SECONDS: self.logger.infoPair("Performing clean-up every", "%f hours" % (self.SECONDS/60.0/60.0)) 39 while True: 40 41 self.config.printDeletionPolicy() 42 self.clean("P2") 43 self.clean("ST") 44 45 self.checkClientStatus() 46 if not self.waitForPollTime(): break 47 48 ''' 49 Finds stuff to delete for this batch type, then: 50 51 - removes from the datastore 52 - deletes from local disk 53 - deletes files from DXLayer 54 55 ''' 56 def clean(self, batchType): 57 58 self.logger.infoSeparator() 59 self.logger.infoTitle("Cleaning up batch type %s" % batchType) 60 61 loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel) 62 loadedDxlayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, self.config.epoch, self.config.dvoLabel) 63 mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel) 64 65 purgedDatastoreIDs = self.ippToPspsDb.getPurgedButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel) 66 purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel) 67 68 self.logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(loadedDatastoreIDs)) 69 self.logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(loadedDxlayerIDs)) 70 self.logger.infoPair("Merged but still on local disk", "%d" % len(mergedLocalIDs)) 71 self.logger.infoPair("Purged but still on datastore", "%d" % len(purgedDatastoreIDs)) 72 self.logger.infoPair("Purged but still on local disk", "%d" % len(purgedLocalIDs)) 40 73 41 if Batch.deleteFromDisk(logger, BASEPATH, batchType, DVOLABEL, id): 42 ippToPspsDb.updateDeletedLocal(id, 1) 43 count = count + 1 74 if self.config.deleteLocal: 75 self.logger.infoSeparator() 76 count = 0 77 ids = mergedLocalIDs + purgedLocalIDs 78 for id in ids: 79 80 if Batch.deleteFromDisk(self.logger, self.config.basePath, batchType, self.config.dvoLabel, id): 81 self.ippToPspsDb.updateDeletedLocal(id, 1) 82 count = count + 1 83 84 self.logger.infoSeparator() 85 self.logger.infoPair("Removed from local disk", "%d" % count) 86 self.logger.infoPair("Remaining on local disk", "%d" % (len(ids) - count)) 87 88 89 if self.config.deleteDatastore: 90 self.logger.infoSeparator() 91 count = 0 92 ids = loadedDatastoreIDs + purgedDatastoreIDs 93 for id in ids: 94 95 batchName = Batch.getNameFromID(id) 96 97 if self.datastore.remove(batchName): 98 count = count + 1 99 100 self.logger.infoSeparator() 101 self.logger.infoPair("Removed from datastore", "%d" % count) 102 self.logger.infoPair("Remaining on datastore", "%d" % (len(ids) - count)) 103 104 if self.config.deleteDxLayer: 105 self.logger.infoSeparator() 106 count = 0 107 for id in loadedDxlayerIDs: 108 109 if self.dxlayer.deleteBatch(id): 110 self.ippToPspsDb.updateDeletedDxlayer(id, 1) 111 count = count + 1 112 113 self.logger.infoSeparator() 114 self.logger.infoPair("Removed from DXLayer", "%d" % count) 115 self.logger.infoPair("Remaining on DXLayer", "%d" % (len(loadedDxlayerIDs) - count)) 116 44 117 45 logger.infoPair("Removed from local disk", "%d" % count) 46 logger.infoPair("Remaining on local disk", "%d" % (len(localIDs) - count)) 47 118 ''' 119 Overrides base-class version 120 ''' 121 def printUsage(self): 122 super(Cleanup, self).printUsage("[<poll time in hours>]") 48 123 49 if DELETEDATASTORE:50 count = 051 for id in datastoreIDs:52 53 batchName = Batch.getNameFromID(id)54 55 if datastore.remove(batchName):56 count = count + 157 58 logger.infoPair("Removed from datastore", "%d" % count)59 logger.infoPair("Remaining on datastore", "%d" % (len(datastoreIDs) - count))60 61 if DELETEDXLAYER:62 count = 063 for id in dxlayerIDs:64 65 if dxlayer.deleteBatch(id):66 ippToPspsDb.updateDeletedDxlayer(id, 1)67 count = count + 168 69 logger.infoPair("Removed from DXLayer", "%d" % count)70 logger.infoPair("Remaining on DXLayer", "%d" % (len(dxlayerIDs) - count))71 72 124 73 125 ''' 74 126 Start of program 75 127 ''' 76 77 if len(sys.argv) > 1: CONFIG = sys.argv[1] 78 else: 79 print "\n** Usage: " + sys.argv[0] + " <configPath> [hours]\n" 80 sys.exit(1) 81 82 if len(sys.argv) > 2: 83 HOURS = float(sys.argv[2]) 84 SECONDS = HOURS * 60.0 * 60.0 85 else: 86 SECONDS = None 87 88 # open config file 89 configDoc = ElementTree(file=CONFIG) 90 91 # set up logging 92 logging.setLoggerClass(PSLogger) 93 logger = logging.getLogger(sys.argv[0]) 94 logger.setup(configDoc, sys.argv[0]) 95 96 ippToPspsDb = IppToPspsDb(logger, configDoc) 97 datastore = Datastore(logger, configDoc, ippToPspsDb) 98 dxlayer = DXLayer(logger) 99 100 DVOLABEL = configDoc.find("dvo/gpc1Label").text 101 EPOCH = configDoc.find("options/epoch").text 102 BASEPATH = configDoc.find("localOutPath").text 103 DELETELOCAL = int(configDoc.find("deletion/local").text) 104 DELETEDATASTORE = int(configDoc.find("deletion/datastore").text) 105 DELETEDXLAYER = int(configDoc.find("deletion/dxlayer").text) 106 107 logger.infoTitle("ippToPsps clean-up") 108 if SECONDS: logger.infoPair("Performing clean-up every", "%f hours" % (SECONDS/60.0/60.0)) 109 logger.infoBool("Deleting from local disk?", DELETELOCAL) 110 logger.infoBool("Deleting from datastore?", DELETEDATASTORE) 111 logger.infoBool("Deleting from DXLayer?", DELETEDXLAYER) 112 113 while True: 114 115 clean("P2") 116 clean("ST") 117 118 if SECONDS: 119 logger.infoPair("Waiting for", "%f hours" % HOURS) 120 time.sleep(SECONDS) 121 else: break 122 128 cleanup = Cleanup(sys.argv) 129 cleanup.run() 130 cleanup.exitProgram("finished")
Note:
See TracChangeset
for help on using the changeset viewer.
