IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Mar 13, 2013, 2:50:08 PM (13 years ago)
Author:
eugene
Message:

tighter control over the cleanup rules : allow items to be blocked for deletion for any of the 3 stages

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/ipptopspsdb.py

    r35217 r35311  
    3030
    3131    '''
    32     Returns a list of merged batch IDs that are merged but not yet deleted
    33     '''
    34     def getMergedButNotDeletedBatchIDs(self, batchType, column):
    35 
     32    Returns a list of merged batch IDs that are merged but not yet deleted (but allowed to be deleted)
     33    '''
     34    def getMergedReadyToDeleteFromLocalDisk(self, batchType):
     35   
    3636        sql = "SELECT DISTINCT batch_id \
    3737               FROM batch \
    … …  
    4040               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
    4141               AND merged = 1 \
    42                AND " + column + " = 0"
    43 
     42               AND deleted_local = 0 \
     43               AND block_delete_local = 0 "
     44   
    4445        ids = []
    4546        try:
    … …  
    4950        except:
    5051            self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
    51 
     52   
    5253        self.logger.debug("Found %d merged but un-deleted items" % len(ids))
    53 
     54   
    5455        return ids
    5556
    5657    '''
    57     Returns a list of batch IDs marked as 'purged' but not yet deleted
    58     '''
    59     def getPurgedButNotDeletedBatchIDs(self, batchType, column):
    60 
     58    Returns a list of batch IDs marked as 'purged' but not yet deleted (but allowed to be deleted)
     59    '''
     60    def getPurgedReadyToDeleteFromLocalDisk(self, batchType):
     61   
    6162        sql = "SELECT DISTINCT batch_id \
    6263               FROM batch \
    … …  
    6566               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
    6667               AND purged = 1 \
    67                AND " + column + " = 0"
    68 
     68               AND deleted_local = 0 \
     69               AND block_delete_local = 0 "
     70   
    6971        ids = []
    7072        try:
    … …  
    7476        except:
    7577            self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
    76 
    77 
     78   
     79   
    7880        self.logger.debug("Found %d merged but un-deleted items" % len(ids))
    79 
     81   
    8082        return ids
    8183
    8284    '''
    83     Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted
    84     '''
    85     def getLoadedToODMButNotDeletedBatchIDs(self, batchType, column):
    86 
     85    Returns a list of batch IDs marked as 'purged' but not yet deleted (but allowed to be deleted)
     86    '''
     87    def getPurgedReadyToDeleteFromDatastore(self, batchType):
     88   
    8789        sql = "SELECT DISTINCT batch_id \
    8890               FROM batch \
    … …  
    9092               AND batch_type = '" + batchType + "' \
    9193               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
     94               AND purged = 1 \
     95               AND deleted_datastore = 0 \
     96               AND block_delete_datastore = 0 "
     97   
     98        ids = []
     99        try:
     100            rs = self.executeQuery(sql)
     101            while (rs.next()): ids.append(rs.getInt(1))
     102            rs.close()
     103        except:
     104            self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
     105   
     106   
     107        self.logger.debug("Found %d merged but un-deleted items" % len(ids))
     108   
     109        return ids
     110
     111    '''
     112    Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted from the datastore
     113    '''
     114    def getLoadedToODMReadyToDeleteFromDatastore(self, batchType):
     115
     116        sql = "SELECT DISTINCT batch_id \
     117               FROM batch \
     118               WHERE timestamp > '" + self.skychunk.epoch + "' \
     119               AND batch_type = '" + batchType + "' \
     120               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
    92121               AND (loaded_to_ODM = -1 OR merge_worthy = 1) \
    93                AND " + column + " = 0"
     122               AND deleted_datastore = 0 AND block_delete_datastore = 0"
    94123
    95124        ids = []
    … …  
    106135
    107136    '''
    108     Returns a list of merged batch IDs that are not deleted from local disk
    109     '''
    110     def getMergedButNotDeletedFromLocalDisk(self, batchType):
    111         return self.getMergedButNotDeletedBatchIDs(batchType, "deleted_local")
    112 
    113     '''
    114     Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from local disk
    115     '''
    116     def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType):
    117         return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_local")
    118 
    119     '''
    120     Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from datastore
    121     '''
    122     def getLoadedToODMButNotDeletedFromDatastore(self, batchType):
    123         return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_datastore")
    124 
    125     '''
    126     Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from DXLayer
    127     '''
    128     def getLoadedToODMButNotDeletedFromDXLayer(self, batchType):
    129         return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_dxlayer")
    130 
    131     '''
    132     Returns a list of purged  batch IDs that not deleted from local disk
    133     '''
    134     def getPurgedButNotDeletedFromLocalDisk(self, batchType):
    135         return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_local")
    136 
    137     '''
    138     Returns a list of purged batch IDs that are not deleted from datastore
    139     '''
    140     def getPurgedButNotDeletedFromDatastore(self, batchType):
    141         return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_datastore")
     137    Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted from the dxlayer
     138    '''
     139    def getLoadedToODMReadyToDeleteFromDxLayer(self, batchType):
     140
     141        sql = "SELECT DISTINCT batch_id \
     142               FROM batch \
     143               WHERE timestamp > '" + self.skychunk.epoch + "' \
     144               AND batch_type = '" + batchType + "' \
     145               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
     146               AND (loaded_to_ODM = -1 OR merge_worthy = 1) \
     147               AND deleted_dxlayer = 0 and block_delete_dxlayer = 0"
     148
     149        ids = []
     150        try:
     151            rs = self.executeQuery(sql)
     152            while (rs.next()): ids.append(rs.getInt(1))
     153            rs.close()
     154        except:
     155            self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
     156
     157        self.logger.debug("Found %d merged but un-deleted items" % len(ids))
     158
     159        return ids
     160
     161    ## '''
     162    ## XXX EAM : removed this function and made it more explicit (with an additional block)
     163    ## Returns a list of merged batch IDs that are merged but not yet deleted
     164    ## '''
     165    ## def getMergedButNotDeletedBatchIDs(self, batchType, column):
     166    ##
     167    ##     sql = "SELECT DISTINCT batch_id \
     168    ##            FROM batch \
     169    ##            WHERE timestamp > '" + self.skychunk.epoch + "' \
     170    ##            AND batch_type = '" + batchType + "' \
     171    ##            AND dvo_db = '" + self.skychunk.dvoLabel + "' \
     172    ##            AND merged = 1 \
     173    ##            AND " + column + " = 0"
     174    ##
     175    ##     ids = []
     176    ##     try:
     177    ##         rs = self.executeQuery(sql)
     178    ##         while (rs.next()): ids.append(rs.getInt(1))
     179    ##         rs.close()
     180    ##     except:
     181    ##         self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
     182    ##
     183    ##     self.logger.debug("Found %d merged but un-deleted items" % len(ids))
     184    ##
     185    ##     return ids
     186
     187    ## '''
     188    ## Returns a list of batch IDs marked as 'purged' but not yet deleted
     189    ## XXX EAM : removed this function and made it more explicit (with an additional block)
     190    ## '''
     191    ## def getPurgedButNotDeletedBatchIDs(self, batchType, column):
     192    ##
     193    ##     sql = "SELECT DISTINCT batch_id \
     194    ##            FROM batch \
     195    ##            WHERE timestamp > '" + self.skychunk.epoch + "' \
     196    ##            AND batch_type = '" + batchType + "' \
     197    ##            AND dvo_db = '" + self.skychunk.dvoLabel + "' \
     198    ##            AND purged = 1 \
     199    ##            AND " + column + " = 0"
     200    ##
     201    ##     ids = []
     202    ##     try:
     203    ##         rs = self.executeQuery(sql)
     204    ##         while (rs.next()): ids.append(rs.getInt(1))
     205    ##         rs.close()
     206    ##     except:
     207    ##         self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
     208    ##
     209    ##
     210    ##     self.logger.debug("Found %d merged but un-deleted items" % len(ids))
     211    ##
     212    ##     return ids
     213
     214    ## '''
     215    ## Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted
     216    ## '''
     217    ## def getLoadedToODMButNotDeletedBatchIDs(self, batchType, column):
     218    ##
     219    ##     sql = "SELECT DISTINCT batch_id \
     220    ##            FROM batch \
     221    ##            WHERE timestamp > '" + self.skychunk.epoch + "' \
     222    ##            AND batch_type = '" + batchType + "' \
     223    ##            AND dvo_db = '" + self.skychunk.dvoLabel + "' \
     224    ##            AND (loaded_to_ODM = -1 OR merge_worthy = 1) \
     225    ##            AND " + column + " = 0"
     226    ##
     227    ##     ids = []
     228    ##     try:
     229    ##         rs = self.executeQuery(sql)
     230    ##         while (rs.next()): ids.append(rs.getInt(1))
     231    ##         rs.close()
     232    ##     except:
     233    ##         self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
     234    ##
     235    ##     self.logger.debug("Found %d merged but un-deleted items" % len(ids))
     236    ##
     237    ##     return ids
     238
     239    # '''
     240    # Returns a list of merged batch IDs that are not deleted from local disk
     241    # '''
     242    # def getMergedButNotDeletedFromLocalDisk(self, batchType):
     243    #     return self.getMergedButNotDeletedBatchIDs(batchType, "deleted_local")
     244
     245    ## '''
     246    ## Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from local disk
     247    ## '''
     248    ## def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType):
     249    ##     return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_local")
     250    ##
     251    ## '''
     252    ## Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from datastore
     253    ## '''
     254    ## def getLoadedToODMButNotDeletedFromDatastore(self, batchType):
     255    ##     return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_datastore")
     256    ##
     257    ## '''
     258    ## Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from DXLayer
     259    ## '''
     260    ## def getLoadedToODMButNotDeletedFromDXLayer(self, batchType):
     261    ##     return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_dxlayer")
     262
     263    ## '''
     264    ## Returns a list of purged  batch IDs that not deleted from local disk
     265    ## '''
     266    ## def getPurgedButNotDeletedFromLocalDisk(self, batchType):
     267    ##     return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_local")
     268    ##
     269    ## '''
     270    ## Returns a list of purged batch IDs that are not deleted from datastore
     271    ## '''
     272    ## def getPurgedButNotDeletedFromDatastore(self, batchType):
     273    ##     return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_datastore")
    142274
    143275
Note: See TracChangeset for help on using the changeset viewer.