IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Sep 6, 2011, 11:00:22 AM (15 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20110710/ippToPsps/jython
Files:
4 deleted
14 edited
16 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20110710/ippToPsps/jython

    • Property svn:ignore set to
      *.class

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/batch.py

    r31840 r32337  
    1111from subprocess import call, PIPE, Popen
    1212
    13 from datastore import Datastore
    1413from scratchdb import ScratchDb
    1514from gpc1db import Gpc1Db
    … …  
    3534    def __init__(self,
    3635                 logger,
     36                 configPath,
    3737                 doc,
    3838                 gpc1Db,
    … …  
    4040                 id,
    4141                 batchType,
    42                  fits,
    43                  survey=""):
     42                 fits):
    4443
    4544        self.everythingOK = False
    4645        self.readHeader = False
     46        self.configPath = configPath
    4747        self.doc = doc
    4848        self.fits = fits
    49         self.header = self.fits.getPrimaryHeader()
    5049
    5150        # set up logging
    … …  
    5352        self.logger.infoSeparator()
    5453        self.logger.debug("Batch class constructor")
     54
     55        # check FITS file is ok - TODO could be neater
     56        if self.fits:
     57            self.header = self.fits.getPrimaryHeader()
     58            if not self.header: return
    5559
    5660        # set up class variables
    … …  
    6064        self.batchType = batchType;
    6165        self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot"
    62         self.survey = survey
    63 
    64         # get dvo info from config
    65         dvoName = self.doc.find("dvo/name").text
     66
     67        # get info from config
     68        self.survey = self.doc.find("options/survey").text
     69        self.pspsSurvey = self.doc.find("options/pspsSurvey").text
     70        self.dvoGpc1Label = self.doc.find("dvo/gpc1Label").text
    6671        self.dvoLocation = self.doc.find("dvo/location").text
    6772        self.useFullTables = int(self.doc.find("dvo/useFullTables").text)
    6873        self.scratchDb = ScratchDb(logger, self.doc, self.useFullTables)
     74
     75        if not self.scratchDb.everythingOK: return
    6976
    7077        # TODO
    … …  
    7582        else:
    7683            self.surveyID = -1;
    77 
    7884       
    7985        # get some options from the config
    … …  
    8187        self.dataRelease = int(self.doc.find("metadata/dataRelease").text)
    8288
    83         # get datastore info from config
    84         self.datastore = Datastore(self.logger)
     89        # create datastore object
     90        self.datastore = Datastore(self.logger, self.doc)
    8591
    8692        # create a new batch
    8793        self.batchID = self.ippToPspsDb.createNewBatch(
    88                 self.getPspsBatchType(),
     94                self.batchType,
    8995                self.id,
    90                 survey,
    91                 dvoName,
     96                self.survey,
     97                self.dvoGpc1Label,
    9298                self.datastore.product)
    9399
    94100        # get local storage location from config
    95         self.batchName = "B%08d" % self.batchID
    96         self.subDir = self.doc.find("localOutPath").text + "/" + self.getPspsBatchType() + "/" + dvoName
    97         self.localOutPath = self.subDir + "/" + self.batchName
     101        self.batchName = Batch.getNameFromID(self.batchID)
     102        self.basePath = self.doc.find("localOutPath").text
     103        self.subDir = Batch.getSubDir(
     104                self.basePath,
     105                self.batchType,
     106                self.dvoGpc1Label)
     107
     108        self.localOutPath = Batch.getOutputPath(
     109                self.basePath,
     110                self.batchType,
     111                self.dvoGpc1Label,
     112                self.batchID)
     113
    98114        if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath)
    99115
    … …  
    105121        if not self.useFullTables: self.scratchDb.createDvoTables()
    106122
    107         self.everythingOK = True
    108 
    109     '''
    110     Destructor
    111     '''
    112     def __del__(self):
    113 
    114         self.logger.debug("Batch destructor")
    115 
    116     '''
    117     Prints metadata to the log
    118     '''
    119     def printMe(self):
    120 
     123        # dump stuff to the log
     124        self.logger.infoTitle("New " + self.batchType + " batch")
    121125        self.logger.infoPair("Batch name", self.batchName)
    122126        self.logger.infoPair("Survey", self.survey)
    123127        self.logger.infoPair("Survey ID", "%d" % self.surveyID)
     128        self.logger.infoPair("Publishing to PSPS as survey", self.pspsSurvey)
    124129        self.logger.infoPair("DVO location", self.dvoLocation)
    125         self.logger.infoPair("Use full DVO tables?", "%d" % self.useFullTables)
     130        self.logger.infoBool("Use full DVO tables?", self.useFullTables)
     131
     132        if self.fits:
     133            self.logger.infoPair("Input FITS file", self.fits.getOriginalPath())
     134            self.logger.infoPair("Input FITS primary header", "%s cards found" % self.fits.getPrimaryHeaderCardCount())
     135
    126136        self.logger.infoPair("Output path", self.localOutPath)
    127137
     138        self.everythingOK = True
     139   
     140    '''
     141    Static method to generated batch name from batch ID
     142    '''
     143    @staticmethod
     144    def getNameFromID(id):
     145       return "B%08d" % id
     146
     147    '''
     148    Static method to generated sub directory for batch on local disk
     149    '''
     150    @staticmethod
     151    def getSubDir(basePath, batchType, dvoLabel):
     152       return basePath + "/" + batchType + "/" + dvoLabel
     153
     154    '''
     155    Static method to generated output path for batch
     156    '''
     157    @staticmethod
     158    def getOutputPath(basePath, batchType, dvoLabel, id):
     159       return Batch.getSubDir(basePath, batchType, dvoLabel) + "/" + Batch.getNameFromID(id)
     160
     161    '''
     162    Static method to get name of tar file
     163    '''
     164    @staticmethod
     165    def getTarFile(id):
     166        return Batch.getNameFromID(id) + ".tar"
     167
     168    '''
     169    Static method to get path of tar file
     170    '''
     171    @staticmethod
     172    def getTarPath(basePath, batchType, dvoLabel, id):
     173        return Batch.getSubDir(basePath, batchType, dvoLabel) + "/" + Batch.getTarFile(id)
     174
     175    '''
     176    Static method to get name of tarball file
     177    '''
     178    @staticmethod
     179    def getTarballFile(id):
     180        return Batch.getTarFile(id) + ".gz"
     181
     182    '''
     183    Static method to get path of tarball file
     184    '''
     185    @staticmethod
     186    def getTarballPath(basePath, batchType, dvoLabel, id):
     187        return Batch.getSubDir(basePath, batchType, dvoLabel) + "/" + Batch.getTarballFile(id)
     188
     189    ''' 
     190    Static method to delete this batch from local disk
     191    '''
     192    @staticmethod
     193    def deleteFromDisk(logger, basePath, batchType, dvoLabel, id):
     194       
     195        tarballPath = Batch.getTarballPath(basePath, batchType, dvoLabel, id)
     196        dirPath = Batch.getOutputPath(basePath, batchType, dvoLabel, id)
     197
     198        if not os.path.exists(tarballPath):
     199            logger.errorPair("Could not find", tarballPath)
     200            return 1
     201
     202        try:
     203            os.remove(tarballPath)
     204            logger.infoPair("Deleted", tarballPath)
     205        except:
     206            logger.errorPair("Could not delete", tarballPath)
     207            return 0
     208   
     209        return 1
     210
     211    '''
     212    Destructor
     213    '''
     214    def __del__(self):
     215
     216        self.logger.debug("Batch destructor")
     217
     218    '''
     219    Looks for a dictionary entry. If it is not there, then replaces it with the supplied default
     220    if, and only if, in test mode
     221    '''
     222    def safeDictionaryAccessWithDefault(self, header, key, default):
     223
     224         value = self.safeDictionaryAccess(header, key)
     225
     226         if value != "NULL": return value
     227         else:
     228             if not self.testMode: return "NULL"
     229             header[key] = default
     230             self.logger.infoPair("Hardcoding " + key + " to", header[key])
     231             return header[key]
     232
    128233    '''
    129234    Returns the string value from this dictionary or else "NULL"
    … …  
    132237
    133238         if key in header: return header[key]
    134          else: return "NULL"
     239         else:
     240             self.logger.errorPair("Missing header field", key)
     241             return "NULL"
    135242
    136243    '''
    … …  
    146253        # batch information
    147254        root.attrib['name'] = self.batchName
    148         root.attrib['type'] = self.getPspsBatchType()
     255        root.attrib['type'] = self.batchType
    149256        root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    150         if self.survey != "":
    151             root.attrib['survey'] = self.getBatchFriendlySurveyType()
     257        if self.batchType != "IN": root.attrib['survey'] = self.pspsSurvey
    152258        try: self.minObjID
    153259        except: pass
    … …  
    158264
    159265        # get md5sum
    160         p = Popen("md5sum " + self.outputFitsPath, shell=True, stdout=PIPE)
    161         p.wait()
    162         out = p.stdout.read()
    163         md5sum = out[0:out.rfind(" ")]
     266        #p = Popen("md5sum " + self.outputFitsPath, shell=True, stdout=PIPE)
     267        #p.wait()
     268        #out = p.stdout.read()
     269        #md5sum = out[0:out.rfind(" ")]
    164270
    165271        # get file size
    166         fileSize = os.path.getsize(self.outputFitsPath)
     272        #fileSize = os.path.getsize(self.outputFitsPath)
    167273
    168274        # file information
    … …  
    170276        root.append(child)
    171277        child.attrib['name'] = self.outputFitsFile
    172         child.attrib['bytes'] = str(fileSize)
    173         child.attrib['md5'] = md5sum
     278        #child.attrib['bytes'] = str(fileSize)
     279        #child.attrib['md5'] = md5sum
    174280
    175281        # now create doc and write to file
    … …  
    187293    Publishes this batch to the datastore
    188294    '''
    189     def publishToDatastore(self):
     295    def tarZipAndpublishToDatastore(self):
    190296     
    191297        # set up filenams and paths
    192         tarFile = self.batchName + ".tar"
    193         tarPath = self.subDir + "/" + tarFile
    194 
    195         tarballFile = tarFile + ".gz"
    196         tarballPath = self.subDir + "/" + tarballFile
     298        tarFile = Batch.getTarFile(self.batchID)
     299        tarPath = Batch.getTarPath(self.basePath, self.batchType, self.dvoGpc1Label, self.batchID)
     300        tarballFile = Batch.getTarballFile(self.batchID)
     301        tarballPath = Batch.getTarballPath(self.basePath, self.batchType, self.dvoGpc1Label, self.batchID)
    197302
    198303        # tar directory
    … …  
    211316
    212317        # now publish to the datastore
    213         if self.datastore.publish(self.batchName, self.subDir, tarballFile, "tgz"):
    214             self.ippToPspsDb.updateLoadedToDatastore(self.batchID, 1)
    215 
    216     '''
    217     Gets PSPS-friendly survey type
    218     '''
    219     def getBatchFriendlySurveyType(self):
    220 
    221         #return "SCR" # TODO
    222         try:
    223             self.survey
    224         except:
    225             return "NA"
    226 
    227         if self.survey == "3PI": return "3PI"
    228         elif self.survey == "MD04": return "MD4"
     318        Batch.publishToDatastore(self.datastore, self.ippToPspsDb, self.batchID, self.batchName, self.subDir, tarballFile)
     319
     320    '''
     321    Static method to publish a batch to the datastore
     322    '''
     323    @staticmethod
     324    def publishToDatastore(datastore, ippToPspsDb, batchID, batchName, subDir, tarballFile):
     325
     326        if datastore.publish(batchName, subDir, tarballFile, "tgz"):
     327            ippToPspsDb.updateLoadedToDatastore(batchID, 1)
    229328        else:
    230             self.logger.error("Don't know this survey: '" + self.survey + "'")
    231             return "NA"
    232 
    233     '''
    234     Gets PSPS friendly batch type TODO only use PSPS batch names, P2, ST etc
    235     '''
    236     def getPspsBatchType(self):
    237 
    238         if self.batchType == "init": return "IN"
    239         elif self.batchType == "detection": return "P2"
    240         elif self.batchType == "stack": return "ST"
    241         else: self.logger.error("Don't know this batch type: " + self.survey)
     329            ippToPspsDb.updateLoadedToDatastore(batchID, -1)
     330           
    242331
    243332    '''
    … …  
    267356            rs.close()
    268357
    269         self.ippToPspsDb.updateMinMaxObjID(self.batchID, self.minObjID, self.maxObjID)
     358        self.ippToPspsDb.updateDetectionStats(self.batchID, self.minObjID, self.maxObjID, self.totalDetections)
    270359        self.logger.infoPair("Total detections", "%ld" % self.totalDetections)
    271360        self.logger.infoPair("Min objID", "%ld" % self.minObjID)
    … …  
    283372             self.tablesToExport.append(table.name)
    284373
    285          self.alterPspsTables();
     374         return self.alterPspsTables();
    286375
    287376    '''
    … …  
    303392    Accepts a regular expression filter so not all tables need to be imported
    304393    '''
    305     def importIppTables(self, filter=""):
     394    def importIppTables(self, columns="*", filter=""):
    306395
    307396      self.logger.infoPair("Importing tables with filter", filter)
    … …  
    313402          match = re.match(filter, table.name)
    314403          if not match: continue
    315           self.logger.infoPair("Reading IPP table", table.name)
     404          self.logger.debugPair("Reading IPP table", table.name)
    316405          table = stilts.tpipe(table, cmd='explodeall')
    317406
    … …  
    321410          # IPP FITS files are littered with infinities, so remove these
    322411          self.logger.debug("Removing Infinity values from all columns")
     412          table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
    323413          table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
    324414          table = stilts.tpipe(table, cmd='replaceval Infinity null *')
    … …  
    330420          except:
    331421              self.logger.exception("Problem writing table '" + table.name + "' to the database")
    332 
    333422
    334423      self.logger.infoPair("Done. Imported", "%d tables" % count)
    … …  
    374463        except:
    375464            self.logger.exception("Could not write to FITS")
     465            self.ippToPspsDb.updateProcessed(self.batchID, -1)
    376466            return False
    377467
    … …  
    400490    '''
    401491    def populatePspsTables(self):
    402         self.logger.warn("Not implemented yet")
     492        self.logger.warn("populatePspsTables not implemented yet")
    403493
    404494    '''
    … …  
    407497    def getIDsFromDVO(self):
    408498
     499        if self.scratchDb.getRowCount("dvoMeta") < 1:
     500            self.logger.error("No DVO IDs found in dvoMeta")
     501            return False
     502
    409503        # TODO path to DVO prog hardcoded temporarily
    410         cmd = "../src/dvograbber " + self.dvoLocation
     504        cmd = "../src/dvograbber " + self.configPath + " " + self.dvoLocation
    411505        self.logger.infoPair("Running DVO", cmd)
    412506        p = Popen(cmd, shell=True, stdout=PIPE)
    413507        p.wait()
    414508        #        out = p.stdout.read()
    415         self.logger.infoPair("DVO access", "complete")
    416 
    417         if self.scratchDb.getRowCount("dvoDetection") < 1:
    418             self.logger.error("No DVO IDs found")
     509
     510        rowCount = self.scratchDb.getRowCount("dvoDetection")
     511        if rowCount < 1:
     512            self.logger.errorPair("DVO error", "No IDs found")
    419513            return False
     514
     515        self.logger.infoPair("DVO access complete. Found", "%d detections" % rowCount)
    420516           
    421517        return True
    422518
    423519    '''
    424     Checks whether this batch has already been processed and published. To be implemented by all subclasses
    425     '''
    426     def alreadyProcessed(self):
    427         self.logger.error("Not implemented")
    428 
    429 
    430     '''
    431520    Creates and publishes a batch
     521    TODO all methdds call below should throw exceptions on failure
    432522    '''
    433523    def run(self):
    434524
    435         if not self.everythingOK: return
    436 
    437         self.createEmptyPspsTables()
     525        if not self.everythingOK:
     526            self.logger.error("Aborting this batch due to (hopefully) previously reported errors")
     527            self.ippToPspsDb.updateProcessed(self.batchID, -1)
     528            return
     529
     530        if not self.createEmptyPspsTables():
     531            self.logger.error("Aborting this batch due to (hopefully) previously reported errors")
     532            self.ippToPspsDb.updateProcessed(self.batchID, -1)
     533            return
     534
    438535        self.importIppTables()
    439         if self.populatePspsTables():
    440             if self.exportPspsTablesToFits():
     536        if not self.populatePspsTables():
     537            self.logger.errorPair("Aborting this batch", "unable to populate PSPS tables")
     538            self.ippToPspsDb.updateProcessed(self.batchID, -1)
     539        else:
     540            if not self.exportPspsTablesToFits():
     541                self.logger.errorPair("Aborting this batch", "unable to export tables to FITS file")
     542            else:
    441543                self.writeBatchManifest()
    442                 if int(self.doc.find("options/publishToDatastore").text): self.publishToDatastore()
     544                if int(self.doc.find("options/publishToDatastore").text): self.tarZipAndpublishToDatastore()
    443545                if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False)
    444546
    445         self.logger.infoPair("Batch......", "complete")
    446         self.logger.infoSeparator()
    447 
    448 
     547from datastore import Datastore
     548

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/datastore.py

    r31839 r32337  
    55import logging
    66import sys
     7import re
    78from xml.etree.ElementTree import ElementTree, Element, tostring
    8 
    99
    1010'''
    … …  
    1717
    1818    '''
    19     def __init__(self, logger):
     19    def __init__(self, logger, doc):
    2020   
    2121        # setup logger
    2222        self.logger = logger
     23        self.doc = doc
    2324        self.logger.debug("Datastore constructor")
    2425
    2526        # open config
    26         doc = ElementTree(file="config.xml")
    2727        self.product = doc.find("datastore/product").text
    2828        self.type = doc.find("datastore/type").text
    … …  
    5151
    5252        if p.returncode != 0:
    53             self.logger.error("Datastore publish failed")
     53            self.logger.errorPair("Datastore publish",  "failed")
    5454            ret = False
    5555        else:
    … …  
    5757            ret = True
    5858           
    59 
    6059        tempFile.close()
    6160        return ret
    6261
     62    '''
     63    Removes an item from the datastore
    6364
    64     '''
    65     Removes an item to the datastore
     65    NB this will return a success if the batch is not found on the datastore
    6666    '''
    6767    def remove(self, name):
    … …  
    7272                   --product " + self.product
    7373
    74         p = Popen(command, shell=True, stdout=PIPE)
     74        p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
    7575        p.wait()
     76        output, errors = p.communicate()
    7677
    7778        if p.returncode != 0:
    78             self.logger.error("Datastore removal of " + name + " failed")
    79             ret = False
     79            if re.search("not found in", errors):
     80                self.logger.infoPair("Datastore batch not found", name)
     81                ret = True
     82            else:
     83                self.logger.infoPair("Datastore removal FAILED", name)
     84                ret = False
    8085        else:
     86            self.logger.infoPair("Datastore removal successful", name)
    8187            ret = True
    82             self.logger.infoPair("Datastore removal successful", name)
    8388           
    8489        return ret
    8590
     91    '''
     92    Removes a range of items from the datastore
     93    '''
     94    def removeRange(self, first, last):
     95
     96        firstInt = int(first[1:])
     97        lastInt = int(last[1:])
     98
     99        for i in range(firstInt, lastInt):
     100           
     101            self.remove(Batch.getNameFromID(i))
     102
     103from batch import Batch

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/detectionbatch.py

    r31848 r32337  
    3030    def __init__(self,
    3131                 logger,
     32                 configPath,
    3233                 configDoc,
    3334                 gpc1Db,
    … …  
    3738       super(DetectionBatch, self).__init__(
    3839               logger,
     40               configPath,
    3941               configDoc,
    4042               gpc1Db,
    4143               ippToPspsDb,
    4244               camID,
    43                "detection",
    44                gpc1Db.getCameraStageSmf(camID),
    45                #"MD04") # TODO
    46                "3PI")
     45               "P2",
     46               gpc1Db.getCameraStageSmf(camID))
    4747
    4848       if not self.everythingOK: return
    … …  
    5050       # get camera meta data
    5151       meta = self.gpc1Db.getCameraStageMeta(self.id)
     52       if not meta:
     53           self.everythingOK = False
     54           return
     55
    5256       self.expID = meta[0];
    5357       self.expName = meta[1];
    … …  
    8084       #self.endY = 8
    8185
     86       # get a fre primary header values. if in test mode, then use defaults
     87       if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL":
     88           self.everythingOK = False
     89           return
     90
     91       if self.safeDictionaryAccessWithDefault(self.header, 'EXPTIME', "1") == "NULL":
     92           self.everythingOK = False
     93           return
     94
     95       if self.safeDictionaryAccessWithDefault(self.header, 'FILTERID', "g.0000") == "NULL":
     96           self.everythingOK = False
     97           return
     98       
    8299       self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0)
    83100     
    84101       # set up some defauts
    85102       self.calibModNum = 0
    86 
    87103       self.totalNumPhotoRef = 0
    88104
    89        # get filterID using init table
     105
    90106       self.filter = self.header['FILTERID'][0:1]
     107       self.filterID = self.scratchDb.getFilterID(self.filter)
    91108
    92109       # insert what we know about this stack batch into the stack table
    93110       self.ippToPspsDb.insertDetectionMeta(self.batchID, self.expID, self.filter)
    94111
    95     '''
    96     Prints metadata to the log
    97     '''
    98     def printMe(self):
    99 
    100         super(DetectionBatch, self).printMe()
    101 
    102         self.logger.infoPair("Cam ID", "%d" % self.id)
    103         self.logger.infoPair("Exp ID", "%d" % self.expID)
    104         self.logger.infoPair("Exp name", "%s" % self.expName)
    105         self.logger.infoPair("Distribution group", "%s" % self.distGroup)
     112       # dump stuff to log
     113       self.logger.infoPair("Cam ID", "%d" % self.id)
     114       self.logger.infoPair("Exp ID", "%d" % self.expID)
     115       self.logger.infoPair("Exp name", "%s" % self.expName)
     116       self.logger.infoPair("Distribution group", "%s" % self.distGroup)
    106117
    107118    '''
    … …  
    164175        ,' ' \
    165176        ,' ' \
    166         ," + self.header['ZPT_ERR'] + " \
    167         ," + self.header['MJD-OBS'] + " \
    168         ," + self.header['EXPREQ'] + " \
    169         ," + self.header['AIRMASS'] + " \
    170         ," + self.header['RA'] + " \
    171         ," + self.header['DEC'] + " \
    172         ,'" + self.header['CTYPE1'] + "' \
    173         ,'" + self.header['CTYPE2'] + "' \
    174         ," + self.header['CRVAL1'] + " \
    175         ," + self.header['CRVAL2'] + " \
    176         ," + self.header['CRPIX1'] + " \
    177         ," + self.header['CRPIX2'] + " \
    178         ," + self.header['CDELT1'] + " \
    179         ," + self.header['CDELT2'] + " \
    180         ," + self.header['PC001001'] + " \
    181         ," + self.header['PC001002'] + " \
    182         ," + self.header['PC002001'] + " \
    183         ," + self.header['PC002002'] + " \
    184         ," + self.header['NPLYTERM'] + " \
    185         ," + self.header['PCA1X3Y0'] + " \
    186         ," + self.header['PCA1X2Y1'] + " \
    187         ," + self.header['PCA1X1Y2'] + " \
    188         ," + self.header['PCA1X0Y3'] + " \
    189         ," + self.header['PCA1X2Y0'] + " \
    190         ," + self.header['PCA1X1Y1'] + " \
    191         ," + self.header['PCA1X0Y2'] + " \
    192         ," + self.header['PCA2X3Y0'] + " \
    193         ," + self.header['PCA2X2Y1'] + " \
    194         ," + self.header['PCA2X1Y2'] + " \
    195         ," + self.header['PCA2X0Y3'] + " \
    196         ," + self.header['PCA2X2Y0'] + " \
    197         ," + self.header['PCA2X1Y1'] + " \
    198         ," + self.header['PCA2X0Y2'] + " \
     177        ," + self.safeDictionaryAccess(self.header, 'ZPT_ERR') + " \
     178        ," + self.safeDictionaryAccess(self.header, 'MJD-OBS') + " \
     179        ," + self.safeDictionaryAccess(self.header, 'EXPREQ') + " \
     180        ," + self.safeDictionaryAccess(self.header, 'AIRMASS') + " \
     181        ," + self.safeDictionaryAccess(self.header, 'RA') + " \
     182        ," + self.safeDictionaryAccess(self.header, 'DEC') + " \
     183        ,'" + self.safeDictionaryAccess(self.header, 'CTYPE1') + "' \
     184        ,'" + self.safeDictionaryAccess(self.header, 'CTYPE2') + "' \
     185        ," + self.safeDictionaryAccess(self.header, 'CRVAL1') + " \
     186        ," + self.safeDictionaryAccess(self.header, 'CRVAL2') + " \
     187        ," + self.safeDictionaryAccess(self.header, 'CRPIX1') + " \
     188        ," + self.safeDictionaryAccess(self.header, 'CRPIX2') + " \
     189        ," + self.safeDictionaryAccess(self.header, 'CDELT1') + " \
     190        ," + self.safeDictionaryAccess(self.header, 'CDELT2') + " \
     191        ," + self.safeDictionaryAccess(self.header, 'PC001001') + " \
     192        ," + self.safeDictionaryAccess(self.header, 'PC001002') + " \
     193        ," + self.safeDictionaryAccess(self.header, 'PC002001') + " \
     194        ," + self.safeDictionaryAccess(self.header, 'PC002002') + " \
     195        ," + self.safeDictionaryAccess(self.header, 'NPLYTERM') + " \
     196        ," + self.safeDictionaryAccess(self.header, 'PCA1X3Y0') + " \
     197        ," + self.safeDictionaryAccess(self.header, 'PCA1X2Y1') + " \
     198        ," + self.safeDictionaryAccess(self.header, 'PCA1X1Y2') + " \
     199        ," + self.safeDictionaryAccess(self.header, 'PCA1X0Y3') + " \
     200        ," + self.safeDictionaryAccess(self.header, 'PCA1X2Y0') + " \
     201        ," + self.safeDictionaryAccess(self.header, 'PCA1X1Y1') + " \
     202        ," + self.safeDictionaryAccess(self.header, 'PCA1X0Y2') + " \
     203        ," + self.safeDictionaryAccess(self.header, 'PCA2X3Y0') + " \
     204        ," + self.safeDictionaryAccess(self.header, 'PCA2X2Y1') + " \
     205        ," + self.safeDictionaryAccess(self.header, 'PCA2X1Y2') + " \
     206        ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y3') + " \
     207        ," + self.safeDictionaryAccess(self.header, 'PCA2X2Y0') + " \
     208        ," + self.safeDictionaryAccess(self.header, 'PCA2X1Y1') + " \
     209        ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y2') + " \
    199210        )"
    200211        self.scratchDb.execute(sql)
    … …  
    357368    Populates the Detection table for this OTA
    358369    '''
    359     def populateDetectionTable(self, ota):
    360 
    361         tableName = "Detection_" + ota
    362        
     370    def populateDetectionTable(self, ota, results):
     371
     372        pspsTableName = "Detection_" + ota
     373        ippTableName = ota + "_psf"
     374       
     375        results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName)
     376
    363377        # drop then re-create table
    364         self.scratchDb.dropTable(tableName)
    365         sql = "CREATE TABLE " + tableName + " LIKE Detection"
     378        self.scratchDb.dropTable(pspsTableName)
     379        sql = "CREATE TABLE " + pspsTableName + " LIKE Detection"
    366380        try: self.scratchDb.execute(sql)
    367381        except: pass
     382       
     383        # delete all detections with PSF_INST_MAG < 17.5, as decreed by Gene
     384        BEFORE = self.scratchDb.getRowCount(ippTableName)
     385        sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5"
     386        self.scratchDb.execute(sql)
     387        #DELETED = BEFORE - self.scratchDb.getRowCount(ippTableName)
     388        results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName)
     389        #self.logger.infoPair("Saturated detections", "%d deleted" % DELETED)
    368390
    369391        # insert all detections into table
    370         sql = "INSERT IGNORE INTO " + tableName + " ( \
     392        sql = "INSERT IGNORE INTO " + pspsTableName + " ( \
    371393               ippDetectID \
     394               ,filterID \
     395               ,surveyID \
     396               ,obsTime \
    372397               ,xPos \
    373398               ,yPos \
    … …  
    380405               ,psfWidMinor \
    381406               ,psfTheta \
     407               ,psfLikelihood \
    382408               ,psfCf \
    383409               ,momentXX \
    … …  
    391417               ,skyErr \
    392418               ,sgSep \
     419               ,activeFlag \
     420               ,assocDate \
     421               ,historyModNum \
     422               ,dataRelease \
    393423               ) \
    394424               SELECT \
    395425               IPP_IDET \
     426               , " + str(self.filterID) + "\
     427               , " + str(self.surveyID) + " \
     428               ," + str(self.obsTime) + " \
    396429               ,X_PSF \
    397430               ,Y_PSF \
    … …  
    404437               ,PSF_MINOR \
    405438               ,PSF_THETA \
     439               ,psfLikelihood(EXT_NSIGMA) \
    406440               ,PSF_QF \
    407441               ,MOMENTS_XX \
    … …  
    415449               ,SKY_SIGMA \
    416450               ,EXT_NSIGMA \
    417                FROM " + ota + "_psf"
    418         self.scratchDb.execute(sql)
    419 
    420         # set obsTime
    421         sql = "UPDATE " + tableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr)
    422         self.scratchDb.execute(sql)
    423         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
    424         self.scratchDb.updateAllRows(tableName, "historyModNum", "0")
    425 
    426         self.scratchDb.updateAllRows(tableName, "surveyID", str(self.surveyID))
    427         self.scratchDb.updateFilterID(tableName, self.filter)
     451               , 0 \
     452               , '" + self.dateStr + "' \
     453               , 0 \
     454               , " + str(self.dataRelease) + "\
     455               FROM " + ippTableName
     456        self.scratchDb.execute(sql)
    428457
    429458        # now delete bad flux and bad chip positions
    430         self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "instFlux")
    431         self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "peakADU")
     459        sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux
     460        self.scratchDb.execute(sql)
     461
     462        # update cosmic ray and extended likelihoods
     463        sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0"
     464        self.scratchDb.execute(sql)
     465        sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0"
     466        self.scratchDb.execute(sql)
     467
     468        # remove detections will NULL inst flux or NULL peak ADU
     469        results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux")
     470        results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU")
    432471
    433472    '''
    … …  
    499538        self.scratchDb.makeColumnPrimaryKey("Detection", "ippDetectID")
    500539
    501     '''
    502     Applies indexes to the IPP tables
    503     '''
    504     def indexIppTables(self):
    505 
    506         self.logger.infoPair("Creating indexes on", "IPP tables")
    507 
    508         for x in range(self.startX, self.endX):
    509             for y in range(self.startY, self.endY):
    510 
    511                 # dodge the corners
    512                 if x==0 and y==0: continue
    513                 if x==0 and y==7: continue
    514                 if x==7 and y==0: continue
    515                 if x==7 and y==7: continue
    516 
    517                 extension = "XY%d%d_psf" % (x, y)
    518                 self.scratchDb.createIndex(extension, "IPP_IDET")
    519 
    520     '''
    521     Updates provided table with DVO IDs from DVO table
    522     '''
    523     def updateDvoIDs(self, table, sourceID, externID):
    524 
    525         imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID)
    526         self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
    527         sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \
    528                a.ippObjID = b.ippObjID, \
    529                a.detectID = b.detectID, \
    530                a.objID = b.objID, \
    531                a.infoFlag = b.flags << 32 | a.infoFlag \
    532                WHERE a.ippDetectID = b.ippDetectID \
    533                AND b.sourceID = " + str(sourceID) + " \
    534                AND b.imageID = " + str(imageID)
    535 
    536         self.scratchDb.execute(sql)
    537 
    538     '''
    539     Generates psf, cosmic ray and extended source likelihoods
    540     '''
    541     def populateLikelihoods(self, tableName):
    542 
    543         sql = "SELECT ippDetectID, psfLikelihood, crLikelihood, extendedLikelihood, sgSep FROM " + tableName
    544         rs = self.scratchDb.executeUpdatableQuery(sql)
    545 
    546         sqrt2 = Math.sqrt(2)
    547 
    548         while rs.next():
    549 
    550             sgSep = rs.getDouble(5)
    551             psfLikelihood = Erf.erfc(Math.abs(sgSep)/sqrt2)
    552        
    553             if sgSep > 0:
    554                 crLikelihood = 0
    555                 extendedLikelihood = 1.0 - psfLikelihood
    556             else:
    557                 crLikelihood = 1.0 - psfLikelihood
    558                 extendedLikelihood = 0
    559 
    560             # update columns
    561             rs.updateDouble(2, psfLikelihood )
    562             rs.updateDouble(3, crLikelihood )
    563             rs.updateDouble(4, extendedLikelihood )
    564 
    565             # now 'commit' to database
    566             rs.updateRow();
    567 
    568 
    569     '''
    570     Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
    571     '''
    572     def populatePspsTables(self):
    573 
    574540        self.populateFrameMeta()
    575541     
    576542        # dictionary objects to hold sourceIDs and imageIDs for later
    577         sourceIDs = {}
    578         imageIDs = {}
     543        self.sourceIDs = {}
     544        self.imageIDs = {}
    579545
    580546        # loop through all OTAs and populate ImageMeta extensions
    … …  
    595561                header = self.fits.findAndReadHeader(ota + ".hdr")
    596562                if not header:
    597                     self.logger.error("No header found for OTA " + ota)
     563                    self.logger.errorPair("No header found for OTA", ota)
    598564                    continue
     565
     566                # check we have valid sourceID/imageID pair from the header
     567                if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": continue
     568                if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": continue
    599569
    600570                # store sourceID/imageID combo in Db so DVO can look up later
    601571                if not self.useFullTables:
    602                     self.scratchDb.insertNewDvoImage(header['SOURCEID'], header['IMAGEID'])
    603 
    604                 # check we have valid sourceID/imageID pair from the header
    605                 if 'SOURCEID' not in header or 'IMAGEID' not in header:
    606                     self.logger.error("Can't read SOURCEID/IMAGEID pair from " + ota + " header")
    607                     continue
     572                    self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID'])
    608573
    609574                # store these for later
    610                 sourceIDs[ota] = header['SOURCEID']
    611                 imageIDs[ota] = header['IMAGEID']
     575                self.sourceIDs[ota] = header['SOURCEID']
     576                self.imageIDs[ota] = header['IMAGEID']
    612577
    613578                # populate ImageMeta
    … …  
    616581             
    617582        # now run DVO code to get all IDs
    618         if not self.useFullTables: self.getIDsFromDVO()
     583        if not self.useFullTables:
     584            if not self.getIDsFromDVO(): return False
     585
     586        return True
     587
     588
     589    '''
     590    Applies indexes to the IPP tables
     591    '''
     592    def indexIppTables(self):
     593
     594        self.logger.infoPair("Creating indexes on", "IPP tables")
     595
     596        for x in range(self.startX, self.endX):
     597            for y in range(self.startY, self.endY):
     598
     599                # dodge the corners
     600                if x==0 and y==0: continue
     601                if x==0 and y==7: continue
     602                if x==7 and y==0: continue
     603                if x==7 and y==7: continue
     604
     605                extension = "XY%d%d_psf" % (x, y)
     606                self.scratchDb.createIndex(extension, "IPP_IDET")
     607
     608    '''
     609    Updates provided table with DVO IDs from DVO table
     610    '''
     611    def updateDvoIDs(self, table, sourceID, externID):
     612
     613        imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID)
     614        self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
     615        sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \
     616               a.ippObjID = b.ippObjID, \
     617               a.detectID = b.detectID, \
     618               a.objID = b.objID, \
     619               a.infoFlag = b.flags << 32 | a.infoFlag \
     620               WHERE a.ippDetectID = b.ippDetectID \
     621               AND b.sourceID = " + str(sourceID) + " \
     622               AND b.imageID = " + str(imageID)
     623
     624        self.scratchDb.execute(sql)
     625
     626
     627    '''
     628    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
     629    '''
     630    def populatePspsTables(self):
    619631
    620632        # loop through all OTAs again to update with DVO IDs
    … …  
    623635        tables = []   
    624636        otaCount = 0
     637        results = {}
     638        totalOriginal = totalSatDet = totalNulIInstFlux = totalNullPeakFlux = totalNullObjID = totalDetections = 0
     639        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
     640        self.logger.info("|  OTA  | Initial total |   Sat Det     | NULL instFlux | NULL peak ADU | NULL obj ID   |  Remainder    |")
     641        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
    625642        for x in range(self.startX, self.endX):
    626643            for y in range(self.startY, self.endY):
    … …  
    633650
    634651                ota = "XY%d%d" % (x, y)
    635                 if ota not in sourceIDs: continue
    636 
    637                 self.logger.infoMiniSeparator()
     652                if ota not in self.sourceIDs: continue
     653
     654                #self.logger.infoTitle("Processing " + ota)
     655                if not self.scratchDb.astrometricSolutionOK(ota):
     656                    self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % ota)
     657                    continue
    638658
    639659                # populate remainder of tables
    640                 self.populateDetectionTable(ota)
     660                self.populateDetectionTable(ota, results)
    641661
    642662                # now add DVO IDs
    643                 self.updateDvoIDs("Detection_" + ota, sourceIDs[ota], imageIDs[ota])
    644                 self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID")
     663                self.updateDvoIDs("Detection_" + ota, self.sourceIDs[ota], self.imageIDs[ota])
     664                results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID")
    645665                self.updateImageID("Detection_" + ota, x, y)
    646                 self.populateLikelihoods("Detection_" + ota)
    647                
    648                 # check we have something in this Detection table
    649                 if self.scratchDb.getRowCount("Detection_" + ota) < 1:
    650                     self.logger.infoPair("Skipping empty table for ota", ota)
     666                rowCount = self.scratchDb.getRowCount("Detection_" + ota)
     667                self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |",
     668                        ota,
     669                        results['ORIGINALTOTAL'],
     670                        results['SATDET'],
     671                        results['NULLINSTFLUX'],
     672                        results['NULLPEAKADU'],
     673                        results['NULLOBJID'],
     674                        rowCount)
     675
     676                totalOriginal = totalOriginal + results['ORIGINALTOTAL']
     677                totalSatDet = totalSatDet + results['SATDET']
     678                totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX']
     679                totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU']
     680                totalNullObjID = totalNullObjID + results['NULLOBJID']
     681                totalDetections = totalDetections + rowCount
     682
     683                # check we have something in this Detection table TODO add this to table above
     684                if rowCount < 1:
     685                    self.logger.debugPair("Skipping empty table for ota", ota)
    651686                    continue;
    652687
    653688                # update ImageMeta with count of detections for this OTA and photoCodeID
    654689                sql = "UPDATE ImageMeta_" + ota + " \
    655                        SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
     690                       SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.sourceIDs[ota], self.imageIDs[ota]))
    656691                self.scratchDb.execute(sql)
    657692
    … …  
    668703                otaCount = otaCount + 1
    669704
     705        # print totals
     706        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
     707        self.logger.info("| Total | %13d | %13d | %13d | %13d | %13d | %13d |",
     708                totalOriginal,
     709                totalSatDet,
     710                totalNulIInstFlux,
     711                totalNullPeakFlux,
     712                totalNullObjID,
     713                totalDetections)
     714        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
     715
    670716        # if we only have one table export, i.e. FrameMeta, then get out of here
    671717        if len(self.tablesToExport) == 1:
    … …  
    708754       if self.testMode: regex = "XY33.psf"
    709755       else : regex = ".*.psf"
    710 
    711        return super(DetectionBatch, self).importIppTables(regex)
     756 
     757       columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS SKY SKY_SIGMA EXT_NSIGMA"
     758
     759       return super(DetectionBatch, self).importIppTables(columns, regex)
    712760
    713761

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/dvoToMySQL.py

    r31398 r32337  
    1010from subprocess import call, PIPE, Popen
    1111
    12 from gpc1db import Gpc1Db
     12from pslogger import PSLogger
    1313from scratchdb import ScratchDb
    1414
    … …  
    2727
    2828    '''
    29     def __init__(self, logger, pathToDvo):
     29    def __init__(self, logger, doc, RESETTABLES):
    3030
    3131        # set up logging
    3232        self.logger = logger
    33         self.pathToDvo = pathToDvo
    34         self.logger.debug("DvoToMySql class constructor")
    35 
    36         self.logger.debug("Important DVO database at " + self.pathToDvo)
    37 
    38         # open config
    39         doc = ElementTree(file="config.xml")
    40 
    41         # create database objects
    42         self.scratchDb = ScratchDb(logger)
    43         self.gpc1Db = Gpc1Db(self.logger)
     33        self.doc = doc
     34        self.logger.infoSeparator()
     35        self.dvoLocation = self.doc.find("dvo/location").text
     36
     37        # create database object
     38        self.scratchDb = ScratchDb(logger, self.doc, 1)
     39
     40        # some log stuff
     41        self.logger.infoPair("Started", "dvoToMySql")
     42        self.logger.infoPair("Importing DVO database at", self.dvoLocation)
    4443
    4544        # create DVO tables
    46         #self.scratchDb.createDvoTables()
     45        if RESETTABLES: self.scratchDb.resetDvoToMysqlTables()
    4746
    4847        # import Images.dat table
    49         sql = "DELETE FROM dvoMetaFull"
    50         self.scratchDb.stmt.execute(sql)
    51 
    52         imagesTableName = self.importFits(self.pathToDvo,
     48        self.logger.infoPair("Deleting from table", self.scratchDb.dvoMetaTable)
     49        sql = "DELETE FROM " + self.scratchDb.dvoMetaTable
     50        self.scratchDb.execute(sql)
     51
     52        imagesTableName = self.importFits(self.dvoLocation,
    5353                "",
    5454                "Images.dat",
    55                 "IMAGE_ID SOURCE_ID CCDNUM EXTERN_ID FLAGS PHOTCODE NSTAR")
    56         self.scratchDb.createIndex(imagesTableName, "IMAGE_ID")
     55                "SOURCE_ID IMAGE_ID CCDNUM EXTERN_ID FLAGS PHOTCODE NSTAR")
     56        self.scratchDb.createIndex(imagesTableName, "EXTERN_ID")
    5757
    5858        # insert into dvoMetaFull
    59         self.logger.info("Inserting all image meta data into database")
    60         sql = "INSERT INTO dvoMetaNew ( \
     59        # NB what we and smf files call IMAGE_ID, DVO calls EXTERN_ID. We are sticking
     60        # with the smf name
     61        self.logger.infoPair("Populating", "all image meta data")
     62        sql = "INSERT INTO " + self.scratchDb.dvoMetaTable + " ( \
    6163               sourceID, \
    6264               imageID, \
    … …  
    7173               PHOTCODE \
    7274               FROM " + imagesTableName
    73         self.scratchDb.stmt.execute(sql)
     75        self.scratchDb.execute(sql)
    7476
    7577        subdirs = ['n0000']
    … …  
    7779        for subdir in subdirs:
    7880
    79             files = glob.glob(pathToDvo + "/" + subdir + "/*.cpm")
     81            files = glob.glob(self.dvoLocation + "/" + subdir + "/*.cpm")
    8082
    8183            #files = ['0247.06', '0244.06', '0244.10']
    … …  
    8587                # get just filename, without extension
    8688                file = os.path.basename(os.path.splitext(file)[0])
    87                 self.logger.info("---------------------------------------------: " + file)
     89                self.logger.infoSeparator()
    8890
    8991                if self.scratchDb.alreadyImportedThisDvoTable(file): continue
    9092
    9193                # import cpm table and index
    92                 cpmTableName = self.importFits(self.pathToDvo,
     94                cpmTableName = self.importFits(self.dvoLocation,
    9395                        subdir,
    9496                        file + ".cpm",
    9597                        "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID DB_FLAGS")
     98                self.scratchDb.createIndex(cpmTableName, "IMAGE_ID")
    9699                self.scratchDb.createIndex(cpmTableName, "CAT_ID")
    97100                self.scratchDb.createIndex(cpmTableName, "OBJ_ID")
    98101
    99102                # import cpt table and index
    100                 cptTableName = self.importFits(self.pathToDvo,
     103                cptTableName = self.importFits(self.dvoLocation,
    101104                        subdir,
    102105                        file + ".cpt",
    103106                        "OBJ_ID CAT_ID EXT_ID")
     107                self.scratchDb.createIndex(cptTableName, "IMAGE_ID")
    104108                self.scratchDb.createIndex(cptTableName, "CAT_ID")
    105109                self.scratchDb.createIndex(cptTableName, "OBJ_ID")
    106110     
    107111                # shove SOURCE_IDs into measurement table
    108                 self.logger.info("Adding SOURCE_IDs into measurements table")
    109                 sql = "ALTER TABLE "+cpmTableName+" ADD COLUMN (SOURCE_ID SMALLINT)"
    110                 self.scratchDb.stmt.execute(sql)
    111                 sql = "UPDATE "+cpmTableName+" AS a, "+imagesTableName+" AS b \
     112                self.logger.infoPair("Adding", "SOURCE_IDs")
     113                sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (SOURCE_ID SMALLINT)"
     114                self.scratchDb.execute(sql)
     115                sql = "UPDATE " + cpmTableName + " AS a, " + imagesTableName + " AS b \
    112116                       SET a.SOURCE_ID = b.SOURCE_ID \
    113117                       WHERE a.IMAGE_ID = b.IMAGE_ID"
    114                 self.scratchDb.stmt.execute(sql)
     118                self.scratchDb.execute(sql)
    115119
    116120                # shove PSPS objID in measurement table
    117                 self.logger.info("Adding PSPS objID into measurements table")
     121                self.logger.infoPair("Adding","PSPS objIDs")
    118122                sql = "ALTER TABLE "+cpmTableName+" ADD COLUMN (PSPS_OBJ_ID BIGINT)"
    119                 self.scratchDb.stmt.execute(sql)
     123                self.scratchDb.execute(sql)
    120124                sql = "UPDATE "+cpmTableName+" AS a, "+cptTableName+" AS b \
    121125                       SET a.PSPS_OBJ_ID = b.EXT_ID \
    122126                       WHERE a.CAT_ID = b.CAT_ID \
    123127                       AND a.OBJ_ID = b.OBJ_ID"
    124                 self.scratchDb.stmt.execute(sql)
    125 
    126                 self.logger.info("Putting everything into dvoDetectionFull table")
    127                 sql = "INSERT IGNORE INTO dvoDetectionFull (\
     128                self.scratchDb.execute(sql)
     129
     130                # NB we use an INSERT IGNORE here. This is because of a known issue where multiple DVO cpm
     131                # files can include the same detection, with the same IMAGE_ID/DET_ID pairing, but different
     132                # PSPS object IDs assigned in the corresponding cpt file. This is believed to be a chip-boundary
     133                # issue within DVO. So, for now, we take the first IMAGE_ID/DET_ID detection we find, and ignore the rest
     134                self.logger.infoPair("Populating", self.scratchDb.dvoDetectionTable)
     135                sql = "INSERT IGNORE INTO " + self.scratchDb.dvoDetectionTable + " (\
    128136                       sourceID \
    129137                       ,imageID \
    … …  
    142150                       ,DB_FLAGS \
    143151                       FROM " + cpmTableName
    144                 self.scratchDb.stmt.execute(sql)
     152                try:
     153                    self.scratchDb.execute(sql)
     154                except:
     155                    self.logger.error("FAILED: " + sql)
     156                    return
    145157
    146158                # now drop what we don't need
    147                 self.logger.info("Dropping tables")
     159                self.logger.infoPair("Dropping table", cpmTableName)
    148160                self.scratchDb.dropTable(cpmTableName)
     161                self.logger.infoPair("Dropping table", cptTableName)
    149162                self.scratchDb.dropTable(cptTableName)
    150163       
    … …  
    173186      tableName = tableName.replace('.', '_')
    174187
    175       self.logger.info("Attempting to import tables from '" + fullPath + "' to '" + tableName + "'")
     188      self.logger.infoPair("Importing tables from file", fullPath)
     189      self.logger.infoPair("Writing to database table", tableName)
    176190
    177191      tables = stilts.treads(fullPath)
    … …  
    180194      for table in tables:
    181195
    182           self.logger.info("Reading IPP table " + table.name + " from FITS file")
     196          self.logger.infoPair("Reading IPP table", table.name)
    183197          table = stilts.tpipe(table, cmd='explodeall')
    184198     
    185199          # IPP FITS files are littered with infinities, so remove these
    186           self.logger.info("Removing Infinity values from all columns")
     200          self.logger.infoPair("Removing", "infinity values")
    187201          table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
    188202          table = stilts.tpipe(table, cmd='replaceval Infinity null *')
    189203
    190204          #try:
    191           self.logger.info("Writing FITS table to database")
     205          self.logger.infoPair("Writing FITS table to", "database")
    192206          table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName)
    193207          #except:
    … …  
    195209          count = count + 1
    196210
    197       self.logger.info("Done. Imported %d tables" % count)
     211      self.logger.infoPair("Finished importing", "%d tables" % count)
    198212
    199213      return tableName
    200214
    201 logging.config.fileConfig("logging.conf")
    202 logger = logging.getLogger("dvotomysql")
    203 logger.info("Starting")
    204 
    205 dvoToMySql = DvoToMySql(logger, "/data/ipp005.0/gpc1/catdirs/MD04.merges/MD04.merge")
    206 #dvoToMySql = DvoToMySql(logger, "/export/ippc00.1/rhenders/MD04.merge")
    207 
    208 logger.info("Program complete")
    209 
     215
     216'''
     217Start of program.
     218'''
     219
     220if len(sys.argv) > 1: CONFIG  = sys.argv[1]
     221else:
     222    print "** Usage: " + sys.argv[0] + " <configPath> [reset]"
     223    sys.exit(1)
     224
     225# open config file
     226configDoc = ElementTree(file=CONFIG)
     227
     228logging.setLoggerClass(PSLogger)
     229logger = logging.getLogger("dvoToMySQL")
     230logger.setup(configDoc, "dvoToMySQL", 0, 1)
     231
     232RESETTABLES = 0
     233
     234if len(sys.argv) > 2 and sys.argv[2] == "reset":
     235    response = raw_input("* Are you ABSOLUTELY sure you want to recreate the DVO tables (y/n)? ")
     236    if response == "y": RESETTABLES = 1
     237    else: sys.exit(1)
     238
     239dvoToMySql = DvoToMySql(logger, configDoc, RESETTABLES)
     240
     241logger.infoPair("Program...", "complete")
     242

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/fits.py

    r31849 r32337  
    1717    def __init__(self, logger, doc, originalPath):
    1818
     19       # set class variables
    1920       self.originalPath = originalPath
    2021       self.logger = logger
    2122       self.doc = doc
     23       self.header = None
    2224       self.localDir = self.doc.find("localOutPath").text
    2325
    24        # does it exist?
     26       # does this file even exist?
    2527       if not os.path.isfile(self.originalPath):
    26            self.logger.error("Cannot read file at '" + self.originalPath + "'")
     28           self.logger.errorPair("Cannot read file", self.originalPath)
    2729           return
    2830
    29        self.logger.infoPair("FITS file", self.originalPath)
     31       # ok, we have a file, now copy it locally to save on NFS overhead
     32       self.logger.debugPair("FITS file", self.originalPath)
    3033       self.localCopyPath = self.localDir + "/temp.fits"
    3134       shutil.copy2(self.originalPath, self.localCopyPath)
    3235
     36       # open the local copy and parse the ridiculou plain-text header
    3337       self.file = open(self.localCopyPath)
    3438       self.header = self.parseHeader()
    35        self.logger.infoPair("FITS local copy", self.localCopyPath)
    36        self.logger.infoPair("FITS primary header", "%d cards found" % len(self.header))
     39       self.logger.debugPair("FITS local copy", self.localCopyPath)
     40       self.logger.debugPair("FITS primary header", "%d cards found" % len(self.header))
     41
     42       # move file pointer back to the start of the file
    3743       self.rewindToStart()
    3844
    3945    '''
    40     Returns the path to this FITS file
     46    Returns count of 'cards' found in primary header
     47    '''
     48    def getPrimaryHeaderCardCount(self):
     49        return len(self.header)
     50
     51    '''
     52    Returns the path to the LOCAL COPY of this FITS file
    4153    '''
    4254    def getPath(self):
    4355        return self.localCopyPath
     56
     57    '''
     58    Returns the path to the LOCAL COPY of this FITS file
     59    '''
     60    def getOriginalPath(self):
     61        return self.originalPath
    4462
    4563    '''
    … …  
    5775        else: return "NULL"
    5876
    59 
     77    '''
     78    Moves the file pointer back to the start of this file
     79    '''
    6080    def rewindToStart(self):
    6181        self.file.seek(0, 0)
    … …  
    6383
    6484    '''
    65     Finds and reads a header extension
     85    Finds and reads a header extension.
     86
     87    2880 is the magic number that the FITS format uses to break up data: extensions always begin on multiples of 2880 bytes
    6688    '''
    6789    def findAndReadHeader(self, name):
    … …  
    91113
    92114            self.file.seek(origIndex, 0)
    93             self.logger.error("...could not read header in extension '" + name + "'")
     115            self.logger.errorPair("Could not read header", name)
    94116            return
    95     #else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards")
     117        else:
     118            self.logger.debug("...read header at '" + name + "' and found " + str(len(header)) + " header cards")
    96119
    97120        return header

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/gpc1db.py

    r31846 r32337  
    6969            sql = "SELECT DISTINCT stage_id \
    7070                   FROM addRun \
    71                    WHERE stage = '" + stage + "' \
    72                    AND dvodb = '" + dvoDb + "'"
     71                   JOIN minidvodbRun USING(minidvodb_name) \
     72                   JOIN minidvodbProcessed USING(minidvodb_id) \
     73                   WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \
     74                   AND minidvodbRun.state = 'merged' \
     75                   AND minidvodbProcessed.fault = 0 \
     76                   AND addRun.stage = '" + stage + "' \
     77                   AND addRun.state = 'full'"
    7378
    7479        elif batchType == "ST":       
    … …  
    7883                   FROM staticskyInput \
    7984                   JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \
    80                    WHERE stage = '" + stage + "' \
    81                    AND dvodb = '" + dvoDb + "'"
    82 
     85                   JOIN minidvodbRun USING(minidvodb_name) \
     86                   JOIN minidvodbProcessed USING(minidvodb_id) \
     87                   WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \
     88                   AND minidvodbRun.state = 'merged' \
     89                   AND minidvodbProcessed.fault = 0 \
     90                   AND addRun.stage = '" + stage + "' \
     91                   AND addRun.state = 'full'"
     92           
    8393        try:
    8494            rs = self.executeQuery(sql)
    … …  
    144154            rs = self.executeQuery(sql)
    145155            rs.first()
     156        except:
     157            self.logger.errorPair("Can't query for", "stack meta data")
     158           
     159        try:
    146160            meta.append(rs.getString(1))
    147161            meta.append(rs.getString(2))
    148162            meta.append(rs.getString(3))
    149163        except:
    150             self.logger.exception("Can't query for stack meta")
    151 
     164            self.logger.errorPair("getStackStageMeta()", "empty meta data")
     165     
    152166        return meta
    153167
    … …  
    168182            rs = self.executeQuery(sql)
    169183            rs.first()
     184        except:
     185            self.logger.errorPair("Can't query for", "camera meta data")
     186
     187        try:
    170188            meta.append(rs.getInt(1))
    171189            meta.append(rs.getString(2))
    … …  
    175193            meta.append(rs.getString(6))
    176194        except:
    177             self.logger.exception("Can't query for camera meta")
     195            self.logger.errorPair("getCameraStageMeta()", "empty meta data")
    178196
    179197        return meta
    … …  
    195213            rs.first()
    196214        except:
    197             self.logger.exception("Can't query for camera smfs")
    198 
     215            self.logger.errorPair("Can't query for smf file for cam_id",  str(camID))
     216            return None
     217
     218        try:
    199219        # get path to base dir of cmf files
    200         path = rs.getString(1)
    201         path = path[0:path.rfind("/")]
     220            path = rs.getString(1)
     221            path = path[0:path.rfind("/")]
     222        except:
     223            self.logger.errorPair("No smf files found for cam_id", str(camID) )
     224            return None
    202225       
    203         # list all cmf files if a neb path
     226        # list all smf files if a neb path
    204227        files = []
    205228        if path.startswith("neb"):
    … …  
    211234        # or not a neb path
    212235        else:
    213             files = glob.glob(path + "/*.cmf")
    214 
    215         if len(files) < 1: return "NULL"
     236            files = glob.glob(path + "/*.smf")
     237
     238        if len(files) < 1: return None
    216239
    217240        return Fits(self.logger, self.doc, files[0]) # TODO just returning first file - check
    … …  
    251274
    252275        self.logger.error("Could not find stack cmf")
    253         return "NULL"
     276        return None
    254277
    255278    '''

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/initbatch.py

    r31287 r32337  
    11#!/usr/bin/env jython
    22
     3import sys
    34import stilts
    45import logging
    … …  
    67from java.lang import *
    78from java.sql import *
     9
     10from xml.etree.ElementTree import ElementTree, Element, tostring
     11
    812from batch import Batch
     13from pslogger import PSLogger
     14from gpc1db import Gpc1Db
     15from ipptopspsdb import IppToPspsDb
     16from stackbatch import StackBatch
     17from detectionbatch import DetectionBatch
    918
    1019'''
    … …  
    1625    Constructor
    1726    '''
    18     def __init__(self, logger):
    19        super(InitBatch, self).__init__(logger, "init")
     27    def __init__(self,
     28            logger,
     29            configPath,
     30            configDoc,
     31            gpc1Db,
     32            ippToPspsDb):
     33       super(InitBatch, self).__init__(logger,
     34               configPath,
     35               configDoc,
     36               gpc1Db,
     37               ippToPspsDb,
     38               0,
     39               "IN",
     40               None)
    2041
    2142       self.outputFitsFile = "00000000.FITS";
    2243       self.outputFitsPath = self.localOutPath + "/" + self.outputFitsFile
    2344
    24 logging.config.fileConfig("logging.conf")
    25 logger = logging.getLogger("initbatch")
    26 initBatch = InitBatch(logger)
    27 initBatch.createEmptyPspsTables()
    28 initBatch.exportPspsTablesToFits()
    29 initBatch.writeBatchManifest()
     45    '''
     46    Empty implementation
     47    '''
     48    def importIppTables(self, filter=""):
     49        self.logger.debug("importIppTables method here to satidfy base-class")
     50
     51
     52    '''
     53    Implementing base class method, though not necessary
     54    '''
     55    def populatePspsTables(self):
     56        self.logger.debug("No processing required for this batch type")
     57        return 1
     58

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/ipptopspsdb.py

    r31842 r32337  
    5757
    5858    '''
    59     TODO
    60     '''
    61     def getUnprocessedIDsForThisStage(self, dvoDb, batchType, startDate):
    62 
    63         if batchType == "P2": # TODO define these someplace
    64 
    65             stage = "cam"
    66             sql = "SELECT DISTINCT stage_id \
    67                    FROM gpc1.addRun \
    68                    WHERE stage = '" + stage + "' \
    69                    AND dvodb = '" + dvoDb + "' \
    70                    AND stage_id NOT IN "
    71 
    72         elif batchType == "ST":
    73 
    74             stage = "staticsky"
    75             sql = "SELECT DISTINCT stack_id \
    76                    FROM gpc1.staticskyInput \
    77                    JOIN gpc1.addRun ON(gpc1.staticskyInput.sky_id = gpc1.addRun.stage_id) \
    78                    WHERE stage = '" + stage + "' \
    79                    AND dvodb = '" + dvoDb + "' \
    80                    AND stack_id NOT IN "
    81 
    82         sql = sql + "(SELECT stage_id \
    83               FROM batch \
    84               WHERE batch_type = '" + batchType + "' \
    85               AND timestamp > '" + startDate + "' \
    86               AND loaded_to_datastore)"
     59    Returns a list of processed batch IDs that are merged but not yet deleted
     60    '''
     61    def getLoadedToODMButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column):
     62
     63        sql = "SELECT DISTINCT batch_id \
     64               FROM batch \
     65               WHERE timestamp > '" + epoch + "' \
     66               AND batch_type = '" + batchType + "' \
     67               AND dvo_db = '" + dvoGpc1Label + "' \
     68               AND loaded_to_ODM != 0 \
     69               AND " + column + " = 0"
    8770
    8871        ids = []
    … …  
    9275                ids.append(rs.getInt(1))
    9376        except:
    94             self.logger.exception("Can't query for ids in DVO")
     77            self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
    9578
    9679        rs.close()
    9780
    98         self.logger.debug("Found %d unprocessed items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage))
     81        self.logger.debug("Found %d merged but un-deleted items" % len(ids))
    9982
    10083        return ids
    10184
    10285    '''
    103     TODO
    104     '''
    105     def getLastBatchPublished(self, batchType):
    106 
    107         sql = "SELECT TIMESTAMPDIFF(MINUTE, timestamp, now())/60.0 \
     86    Returns a list of processed batch IDs that are merged but not deleted from local disk
     87    '''
     88    def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label):
     89        return self.getLoadedToODMButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_local")
     90
     91    '''
     92    Returns a list of processed batch IDs that are merged but not deleted from datastore
     93    '''
     94    def getLoadedToODMButNotDeletedFromDatastore(self, batchType, epoch, dvoGpc1Label):
     95        return self.getLoadedToODMButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_datastore")
     96
     97    '''
     98    Returns a list of processed batch IDs that are merged but not deleted from DXLayer
     99    '''
     100    def getLoadedToODMButNotDeletedFromDXLayer(self, batchType, epoch, dvoGpc1Label):
     101        return self.getLoadedToODMButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_dxlayer")
     102
     103
     104    '''
     105    Returns a list of processed batch IDs that are not loaded to the datastore
     106    '''
     107    def getProcessedButFailedDatastoreBatchIDs(self, epoch, dvoGpc1Label, batchType):
     108
     109        sql = "SELECT DISTINCT batch_id \
     110               FROM batch \
     111               WHERE timestamp > '" + epoch + "' \
     112               AND batch_type = '" + batchType + "' \
     113               AND dvo_db = '" + dvoGpc1Label + "' \
     114               AND processed = 1\
     115               AND loaded_to_datastore = -1"
     116
     117        ids = []
     118        try:
     119            rs = self.executeQuery(sql)
     120            while (rs.next()):
     121                ids.append(rs.getInt(1))
     122        except:
     123            self.logger.exception("Can't query for processed batch ids not loaded to datastore ipptopsps Db")
     124
     125        rs.close()
     126
     127        self.logger.debug("Found %d processed but-not-on-datastore items" % len(ids))
     128
     129        return ids
     130
     131    '''
     132    Returns a list of processed batch IDs that have not yet been loaded to the ODM
     133    '''
     134    def getBatchIDsUnloadedToODM(self, epoch, dvoGpc1Label):
     135
     136        sql = "SELECT DISTINCT batch_id \
     137               FROM batch \
     138               WHERE timestamp > '" + epoch + "' \
     139               AND dvo_db = '" + dvoGpc1Label + "' \
     140               AND loaded_to_datastore = 1 \
     141               AND loaded_to_ODM = 0"
     142
     143
     144        ids = []
     145        try:
     146            rs = self.executeQuery(sql)
     147            while (rs.next()):
     148                ids.append(rs.getInt(1))
     149        except:
     150            self.logger.exception("Can't query for unloaded batch ids in ipptopsps Db")
     151
     152        rs.close()
     153
     154        self.logger.debug("Found %d unloaded items" % len(ids))
     155
     156        return ids
     157
     158    '''
     159    Returns a list of processed batch IDs that have not failed to load and are not yet merged, for this epoch and dvo label
     160    '''
     161    def getUnfinishedBatchIDs(self, epoch, dvoGpc1Label):
     162
     163        sql = "SELECT DISTINCT batch_id \
     164               FROM batch \
     165               WHERE timestamp > '" + epoch + "' \
     166               AND dvo_db = '" + dvoGpc1Label + "' \
     167               AND loaded_to_datastore = 1 \
     168               AND merged != 1 \
     169               AND loaded_to_ODM != -1"
     170
     171
     172        ids = []
     173        try:
     174            rs = self.executeQuery(sql)
     175            while (rs.next()):
     176                ids.append(rs.getInt(1))
     177        except:
     178            self.logger.exception("Can't query for unfinished batch ids in ipptopsps Db")
     179
     180        rs.close()
     181
     182        self.logger.debug("Found %d unfinished items" % len(ids))
     183
     184        return ids
     185
     186    '''
     187    Returns a list of batch IDs for this epoch and dvo label for which the specified column is set to value
     188    '''
     189    def getBatchIDs(self, epoch, dvoGpc1Label, column, value):
     190
     191        sql = "SELECT DISTINCT batch_id \
     192               FROM batch \
     193               WHERE timestamp > '" + epoch + "' \
     194               AND dvo_db = '" + dvoGpc1Label + "' \
     195               AND " + column + " = " + str(value)
     196
     197        ids = []
     198        try:
     199            rs = self.executeQuery(sql)
     200            while (rs.next()):
     201                ids.append(rs.getInt(1))
     202        except:
     203            self.logger.exception("Can't query for batch ids with " + column + " = " + str(value) + " in ipptopsps Db")
     204
     205        rs.close()
     206
     207        self.logger.debug("Found %d batches with %s = %d" % (len(ids), column, value))
     208
     209        return ids
     210
     211    '''
     212    Returns a list of IDs for this batch type, epoch and dvo label (i.e. either
     213            cam_ids or stack_ids) for which the specified column is set to value
     214    '''
     215    def getStageIDs(self, batchType, epoch, dvoGpc1Label, column, value):
     216
     217        sql = "SELECT DISTINCT stage_id \
    108218               FROM batch \
    109219               WHERE batch_type = '" + batchType + "' \
    110                AND loaded_to_datastore \
     220               AND timestamp > '" + epoch + "' \
     221               AND dvo_db = '" + dvoGpc1Label + "' \
     222               AND " + column + " = " + str(value)
     223
     224        ids = []
     225        try:
     226            rs = self.executeQuery(sql)
     227            while (rs.next()):
     228                ids.append(rs.getInt(1))
     229        except:
     230            self.logger.exception("Can't query for ids with " + column + " = " + str(value) + " in ipptopsps Db")
     231
     232        rs.close()
     233
     234        self.logger.debug("Found %d batches with %s = 1 for batch type='%s'" % (len(ids), column, batchType))
     235
     236        return ids
     237
     238    '''
     239    Series of getter methods that utilise getBatchIDs() method above
     240    '''
     241    def getloadedToDatastoreBatchIDs(self, epoch, dvoGpc1Label):
     242        return self.getBatchIDs(epoch, dvoGpc1Label, "loaded_to_datastore", 1)
     243    def getFailedLoadedToODMBatchIDs(self, epoch, dvoGpc1Label):
     244        return self.getBatchIDs(epoch, dvoGpc1Label, "loaded_to_ODM", -1)
     245
     246    '''
     247    Series of getter methods that utilise getStageIDs() method above
     248    '''
     249    def getProcessedIDs(self, batchType, epoch, dvoGpc1Label):
     250        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "processed", 1)
     251    def getloadedToDatastoreIDs(self, batchType, epoch, dvoGpc1Label):
     252        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_datastore", 1)
     253    def getLoadedToODMIDs(self, batchType, epoch, dvoGpc1Label):
     254        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_ODM", 1)
     255    def getMergeWorthyIDs(self, batchType, epoch, dvoGpc1Label):
     256        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merge_worthy", 1)
     257    def getMergedIDs(self, batchType, epoch, dvoGpc1Label):
     258        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merged", 1)
     259
     260    '''
     261    Returns the time (as a string) that the last batch was published to the datastore for
     262    this epoch, dvo label and batch type
     263    '''
     264    def getTimeOfLastBatchPublished(self, batchType, epoch, dvoGpc1Label):
     265
     266        minutes = None
     267
     268        sql = "SELECT TIMESTAMPDIFF(MINUTE, timestamp, now()) \
     269               FROM batch \
     270               WHERE batch_type = '" + batchType + "' \
     271               AND loaded_to_datastore = 1 \
     272               AND timestamp > '" + epoch + "' \
     273               AND dvo_db = '" + dvoGpc1Label + "' \
    111274               ORDER BY timestamp DESC LIMIT 1"
    112275
    113276        try:
    114277            rs = self.executeQuery(sql)
    115             rs.first()
    116             hours = rs.getFloat(1)
     278            if rs.first(): minutes = rs.getFloat(1)
    117279        except:
    118280            self.logger.exception("Unable to get last batch published")
    119281
    120         return hours
     282        if not minutes: return "Never"
     283
     284        hours = minutes/60.0
     285        days = hours/24.0
     286        weeks = days/7.0
     287
     288        if minutes < 60: return "%.1f minutes ago" % minutes
     289        elif hours < 48: return "%.1f hours ago" % hours
     290        elif days < 7: return "%.1f days ago" % days
     291        return "%.1f weeks ago" % weeks
    121292
    122293    '''
    … …  
    172343        return bpm
    173344
    174 
    175     '''
    176     TODO
    177     '''
    178     def getTotalFailedBatches(self, batchType, startTime, endTime=""):
    179 
    180         sql = "SELECT COUNT(DISTINCT stage_id) \
    181                FROM batch \
    182                WHERE timestamp > '" + startTime + "' \
    183                AND batch_type = '" + batchType + "' \
    184                AND !processed \
    185                AND stage_id NOT IN \
    186                (SELECT DISTINCT stage_id \
    187                 FROM batch \
    188                 WHERE timestamp > '" + startTime + "'  \
    189                 AND batch_type = '" + batchType + "' \
    190                 AND loaded_to_datastore)"
    191 
    192         try:
    193             rs = self.executeQuery(sql)
    194             rs.first()
    195             total = rs.getInt(1)
    196         except:
    197             self.logger.exception("Unable to count failed batches")
    198 
    199         return total
    200     '''
    201     TODO
    202     '''
    203     def getTotalBatchesPublished(self, batchType, startTime, endTime=""):
    204 
    205         sql = "SELECT COUNT(*) \
    206                FROM batch \
    207                WHERE timestamp > '" + startTime + "' \
    208                AND batch_type = '" + batchType + "' \
    209                AND loaded_to_datastore"
    210 
    211         try:
    212             rs = self.executeQuery(sql)
    213             rs.first()
    214             total = rs.getInt(1)
    215         except:
    216             self.logger.exception("Unable to count batches")
    217 
    218         return total
    219 
    220345    '''
    221346    Updates min/max object ID on this table and batch
    222347    '''
    223     def updateMinMaxObjID(self, batchID, minObjID, maxObjID):
     348    def updateDetectionStats(self, batchID, minObjID, maxObjID, totalDetections):
    224349
    225350        sql = "UPDATE batch SET \
    226351               min_obj_id = " + str(minObjID) + ", \
    227                max_obj_id = " + str(maxObjID) + " \
     352               max_obj_id = " + str(maxObjID) + ", \
     353               total_detections = " + str(totalDetections) + " \
    228354               WHERE batch_id = " + str(batchID)
    229355
    … …  
    231357
    232358    '''
     359    Updates a batch-table column
     360    '''
     361    def updateColumn(self, batchID, column, value):
     362
     363        sql = "UPDATE batch \
     364               SET " + column + " = " + str(value) + " \
     365               WHERE batch_id = " + str(batchID)
     366
     367        self.execute(sql)
     368
     369    '''
    233370    Updates batch processed field
    234371    '''
    235372    def updateProcessed(self, batchID, processed):
    236 
     373        self.updateColumn(batchID, "processed", processed)
     374
     375    '''
     376    Updates batch deleted local field
     377    '''
     378    def updateDeletedLocal(self, batchID, value):
     379        self.updateColumn(batchID, "deleted_local", value)
     380
     381    '''
     382    Updates batch deleted datastore field
     383    '''
     384    def updateDeletedDatastore(self, batchID, value):
     385        self.updateColumn(batchID, "deleted_datastore", value)
     386
     387    '''
     388    Updates batch deleted dxlayer field
     389    '''
     390    def updateDeletedDxlayer(self, batchID, value):
     391        self.updateColumn(batchID, "deleted_dxlayer", value)
     392
     393    '''
     394    Updates batch loaded_to_datastore field
     395    '''
     396    def updateLoadedToDatastore(self, batchID, loadedToDatastore):
     397        self.updateColumn(batchID, "loaded_to_datastore", loadedToDatastore)
     398
     399    '''
     400    Updates ODM status for this batch
     401    '''
     402    def updateOdmStatus(self, batchID, odmStatus):
     403
     404        if odmStatus['LOADFAILED'] == 1: loadedToODM = -1
     405        else: loadedToODM = odmStatus['LOADEDTOODM']
    237406        sql = "UPDATE batch \
    238                SET processed = " + str(processed) + " \
     407               SET loaded_to_ODM = " + str(loadedToODM) + ", \
     408               merge_worthy = " + str(odmStatus['MERGEWORTHY']) + ", \
     409               merged = " + str(odmStatus['MERGED']) + " \
    239410               WHERE batch_id = " + str(batchID)
    240411
    … …  
    242413
    243414    '''
    244     Updates batch loaded_to_datastore field
    245     '''
    246     def updateLoadedToDatastore(self, batchID, loadedToDatastore):
    247 
    248         sql = "UPDATE batch \
    249                SET loaded_to_datastore = " + str(loadedToDatastore) + " \
    250                WHERE batch_id = " + str(batchID)
    251 
    252         self.execute(sql)
    253 
    254     '''
    255415    Have we already processed and published this batch?
    256416    '''
    257     def alreadyProcessed(self, stage_id):
     417    def alreadyProcessed(self, batchType, stage_id, epoch, dvoGpc1Label):
    258418
    259419        sql = "SELECT COUNT(*) \
    260420               FROM batch \
    261421               WHERE stage_id = " + str(stage_id) + " \
    262                AND processed \
    263                AND loaded_to_datastore"
     422               AND batch_type = '" + batchType + "' \
     423               AND timestamp > '" + epoch + "' \
     424               AND dvo_db = '" + dvoGpc1Label + "' \
     425               AND processed = 1 \
     426               AND loaded_to_datastore != 0"
    264427
    265428        try:
    … …  
    267430            rs.first()
    268431            if rs.getInt(1) > 0:
    269                 self.logger.error("Batch with stage_id = "+str(stage_id) + " has already been processed and published to datastore")
     432                self.logger.errorPair(str(stage_id) + " has already been published", "skipping")
    270433                return True
    271434            else:
    … …  
    273436        except:
    274437            self.logger.exception("Unable to check whether this batch has already been processed")
     438
     439    '''
     440    Has this stage_id consistently failed to process?
     441    '''
     442    def consistentlyFailed(self, batchType, stage_id, epoch, dvoGpc1Label, count):
     443
     444        sql = "SELECT COUNT(*) \
     445               FROM batch \
     446               WHERE stage_id = " + str(stage_id) + " \
     447               AND batch_type = '" + batchType + "' \
     448               AND timestamp > '" + epoch + "' \
     449               AND dvo_db = '" + dvoGpc1Label + "' \
     450               AND processed = -1"
     451
     452        try:
     453            rs = self.executeQuery(sql)
     454            rs.first()
     455            if rs.getInt(1) >= count:
     456                self.logger.errorPair(str(stage_id) + " has failed %d times" % count, "skipping")
     457                return True
     458            else:
     459                return False
     460        except:
     461            self.logger.exception("Unable to check whether this batch has consistently failed")
    275462
    276463     

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/mysql.py

    r31844 r32337  
    3636        # set up JDBC connection
    3737        self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?user="+self.dbUser+"&password="+self.dbPass
    38         self.con = DriverManager.getConnection(self.url)
    39         self.connectionID = self.getLastConnectionID()
    40         self.logger.debug("MySQL connection to %s with ID %d" % (dbType, self.connectionID))
    41 
    42         #self.stmt = self.con.createStatement()
     38        try:
     39            self.con = DriverManager.getConnection(self.url)
     40            self.connectionID = self.getLastConnectionID()
     41            self.logger.debug("MySQL connection to %s with ID %d" % (dbType, self.connectionID))
     42        except:
     43            self.logger.error("Unable to connect to " + self.url)
     44            self.everythingOK = False
     45            return
     46
     47        self.everythingOK = True
    4348
    4449
    … …  
    210215        nBad = rs.getInt(1)
    211216
    212         sql="DELETE from " + tableName + " WHERE " + columnName + " = " + value
    213         self.execute(sql)
    214         self.logger.info("%s '%s' values in %s    %5d deleted" % (columnName, value, tableName, nBad))
     217        sql="DELETE FROM " + tableName + " WHERE " + columnName + " = " + value
     218        self.execute(sql)
     219        self.logger.debugPair("%s rows with %s" % (columnName, value), "%d deleted" % nBad)
     220
     221        return nBad
    215222
    216223    '''
    … …  
    224231        nBad = rs.getInt(1)
    225232
    226         sql="DELETE from " + tableName + " WHERE " + columnName + " IS NULL"
    227         self.execute(sql)
    228         self.logger.infoPair("NULLs deleted from %s" % tableName, "%-5d %s" % (nBad, columnName))
     233        sql="DELETE FROM " + tableName + " WHERE " + columnName + " IS NULL"
     234        self.execute(sql)
     235        self.logger.debugPair("NULL %s values" % columnName, "%d deleted" % nBad)
     236
     237        return nBad
    229238
    230239    '''

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/pslogger.py

    r31843 r32337  
    11import logging.config
    2 
     2import datetime
     3import os
     4import sys
    35
    46class PSLogger(logging.getLoggerClass()):
     7
     8   '''
     9   Sets up logging using values from the ippToPsps config file
     10   '''
     11   def setup(self, doc, name, stdout=1, sendToFile=0):
     12
     13       formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S')
     14       self.setLevel(logging.INFO)
     15
     16       if sendToFile:
     17           PATH = doc.find("localOutPath").text + "/log"
     18           if not os.path.exists(PATH): os.makedirs(PATH)
     19           DVOLABEL = doc.find("dvo/gpc1Label").text
     20           FULLPATH = PATH + "/" + name + "_" + DVOLABEL + ".log"
     21
     22           # opens file to be appended
     23           hdlr = logging.FileHandler(FULLPATH, "a")
     24           hdlr.setFormatter(formatter)
     25           self.addHandler(hdlr)
     26
     27       if stdout:
     28           stdout = logging.StreamHandler(sys.__stdout__)
     29           #stdout.setFormatter(formatter)
     30           self.addHandler(stdout)
     31
     32       if sendToFile: print "Writing log to: " + FULLPATH
    533
    634   def infoPair(self, first, second):
    735       self.info("%-40s%s" % (first, second))
    836
     37   def errorPair(self, first, second):
     38       self.error("%-40s%s" % (first, second))
     39
     40   def debugPair(self, first, second):
     41       self.debug("%-40s%s" % (first, second))
     42
     43   def infoTitle(self, str):
     44       self.info("")
     45       self.info("%40s" % (str))
     46       self.info("")
     47
     48   def infoBool(self, first, bool):
     49       if bool: str = "yes"
     50       else: str = "no"
     51       self.info("%-40s%s" % (first, str))
     52
     53   def errorBool(self, first, bool):
     54       if bool: str = "yes"
     55       else: str = "no"
     56       self.error("%-40s%s" % (first, str))
     57
    958   def infoSeparator(self):
    1059       self.info("--------------------------------------------------------------------------")
    1160
    12    def infoMiniSeparator(self):
    13        self.info("---------------------------")
    14 

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/removeFromDatastore.py

    r31289 r32337  
    55import sys
    66import getopt
     7from pslogger import PSLogger
     8
     9from xml.etree.ElementTree import ElementTree, Element, tostring
    710
    811
    9 logging.config.fileConfig("logging.conf")
    10 logger = logging.getLogger("datastore")
     12if len(sys.argv) < 3:
     13    print "** Usage: " + sys.argv[0] + " <configPath> <firstBatchToRemove> [<lastBatchToRemove>]"
     14    sys.exit(1)
    1115
     16CONFIG = sys.argv[1]
     17configDoc = ElementTree(file=CONFIG)
    1218
     19logging.setLoggerClass(PSLogger)
     20logger = logging.getLogger("removeFromDatastore")
     21logger.setup(configDoc, "removeFromDatastore")
    1322
     23datastore = Datastore(logger, configDoc)
    1424
    15 datastore = Datastore(logger)
    16 datastore.remove(str(sys.argv[1]))
     25if len(sys.argv) < 4:
     26    datastore.remove(str(sys.argv[2]))
     27else:
     28    datastore.removeRange(str(sys.argv[2]), str(sys.argv[3]))
    1729

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/scratchdb.py

    r31841 r32337  
    2323
    2424        if useFull:
    25             self.dvoMeta = "dvoMetaNew"
    26             self.dvoDetection = "dvoDetectionFull"
     25            self.dvoMetaTable = "dvoMetaFull"
     26            self.dvoDetectionTable = "dvoDetectionFull"
     27            self.dvoDoneTable = "dvoDone"
    2728        else:
    28             self.dvoMeta = "dvoMeta"
    29             self.dvoDetection = "dvoDetection"
    30 
    31         self.logger.debug("ScratchDb constructor, using DVO tables: " + self.dvoMeta + " and " + self.dvoDetection)
     29            self.dvoMetaTable = "dvoMeta"
     30            self.dvoDetectionTable = "dvoDetection"
     31
     32        self.logger.debugPair("Using DVO meta table", self.dvoMetaTable)
     33        self.logger.debugPair("Using DVO detection table", self.dvoDetectionTable)
    3234
    3335    '''
    … …  
    3739
    3840        self.logger.debug("ScratchDb destructor")
     41
     42
     43    '''
     44    Creates a stored procedure for psfLikelihood. This uses an implementation of the complimentary error function taken from Numerical Recipes in C.
     45    '''
     46    def createPsfLikelihoodFunction(self):
     47        sql = "DELIMITER $$ \
     48               DROP FUNCTION IF EXISTS `psfLikelihood` $$ \
     49               CREATE FUNCTION `psfLikelihood`(x REAL) RETURNS REAL \
     50               DETERMINISTIC \
     51               COMMENT 'Returns the psfLikelihood using the complimentary error function with fractional error everywhere less than 1.2 x 10-7. From Numerical Recipes in C' \
     52               BEGIN \
     53               DECLARE ans REAL; \
     54               DECLARE t REAL; \
     55               DECLARE z REAL; \
     56               SET x = ABS(x) / 1.4142135623731; \
     57               SET z=ABS(x); \
     58               SET t=1.0 / (1.0 + 0.5*z); \
     59               SET ans = t * EXP(-z*z-1.26551223+t*(1.00002368+t*(.37409196+t*(.09678418+t*(-.18628806+t*(.27886807+t*(-1.13520398+t*(1.48851587+t*(-.82215223+t*.17087277))))))))); \
     60               IF x < 0.0 THEN \
     61               SET ans = 2.0 - ans; \
     62               END IF; \
     63               RETURN ans; \
     64               END $$ \
     65               DELIMITER ;"
     66
     67        try:
     68            self.execute(sql)
     69            self.logger.debugPair("Installed stored procedure", "psfLikelihood")
     70            return True
     71        except:
     72            self.logger.errorPair("Could not install stored procedure", "psfLikelihood")
     73            return False
    3974
    4075    '''
    … …  
    4782            rs = self.executeQuery(sql)
    4883            rs.first()
    49             return rs.getInt(1)
    50         except:
    51             self.logger.exception("No survey ID found for this survey: '" + self.survey + "'")
    52             return -1
     84            surveyID = rs.getInt(1)
     85        except:
     86            self.logger.errorPair("No survey ID found for survey", name)
     87            surveyID = -1
     88
     89        return surveyID
     90
     91    '''
     92    Gets filter ID for this filter name
     93    '''
     94    def getFilterID(self, name):
     95     
     96        sql = "SELECT filterID FROM Filter WHERE filterType = '" + name + "'"
     97        try:
     98            rs = self.executeQuery(sql)
     99            rs.first()
     100            filterID = rs.getInt(1)
     101        except:
     102            self.logger.errorPair("No filter ID found for filter", name)
     103            filterID = -1
     104
     105        return filterID
     106
     107    '''
     108    Gets imageID from extern ID
     109    '''
     110    def getImageIDFromExternID(self, sourceID, externID):
     111               
     112        imageID = -1
     113               
     114        sql = "SELECT imageID FROM " + self.dvoMetaTable + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
     115        try:
     116            rs = self.executeQuery(sql)   
     117            rs.first()
     118            imageID = rs.getInt(1)
     119        except:
     120            self.logger.errorPair("Unable to get imageID from dvo using", sql)
     121             
     122        return imageID
    53123
    54124    '''
    … …  
    59129        flags = 0
    60130
    61         sql = "SELECT flags FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
     131        sql = "SELECT flags FROM " + self.dvoMetaTable + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
    62132        try:
    63133            rs = self.executeQuery(sql) 
    … …  
    65135            flags = rs.getInt(1)
    66136        except:
    67             self.logger.exception("Unable to get flags from dvo table using " + sql)
     137            self.logger.errorPair("Unable to get flags from dvo using", sql)
    68138        return flags
    69139
    70140    '''
    71     Gets imageID from extern ID
    72     '''
    73     def getImageIDFromExternID(self, sourceID, externID):
    74 
    75         imageID = -1
    76 
    77         sql = "SELECT imageID FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
     141    Gets photcode (aka photoCalID from dvo table)
     142    '''
     143    def getPhotoCalID(self, sourceID, externID):
     144
     145        photcode = -1
     146
     147        sql = "SELECT photcode FROM " + self.dvoMetaTable + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
    78148        try:
    79149            rs = self.executeQuery(sql) 
    80150            rs.first()
    81             imageID = rs.getInt(1)
    82         except:
    83             self.logger.exception("Unable to get imageID from dvo meta table using " + sql)
    84 
    85         return imageID
    86 
    87     '''
    88     Gets photcode (aka photoCalID from dvo table)
    89     '''
    90     def getPhotoCalID(self, sourceID, externID):
    91 
    92         photcode = -1
    93 
    94         sql = "SELECT photcode FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
    95         try:
    96             rs = self.executeQuery(sql) 
    97             rs.first()
    98151            photcode = rs.getInt(1)
    99152        except:
    100             self.logger.exception("Unable to get photcode from dvo table with: " + sql)
     153            self.logger.errorPair("Unable to get photcode from dvo with", sql)
    101154
    102155        return photcode
    … …  
    113166    Inserts a new sourceID/imageID combo into dvoMeta
    114167    '''
    115     def insertNewDvoImage(self, sourceID, imageID):
     168    def insertNewDvoExternID(self, sourceID, externID):
    116169
    117170        sql = "INSERT INTO dvoMeta ( \
    118171               sourceID, \
    119                imageID, \
    120172               externID \
    121173               ) VALUES (\
    122174               " + str(sourceID) + ", \
    123                " + str(imageID) + " ,   \
    124                " + str(imageID) + "    \
     175               " + str(externID) + "   \
    125176               )"
    126177        self.execute(sql)
    … …  
    135186       
    136187    '''
     188    Checks whether the astrometric solution is ok for this chip
     189    TODO the value of 50 for numAstroRef is hardcoded here, but this should be temporary anyway
     190    '''
     191    def astrometricSolutionOK(self, ota):
     192
     193        sql = "SELECT numAstroRef FROM ImageMeta_" + ota
     194
     195        try:
     196            rs = self.executeQuery(sql)
     197            rs.first()
     198            if rs.getInt(1) < 50:
     199                self.logger.debug("Bad astrometric solution for",  ota)
     200                return False
     201            else:
     202                return True
     203        except:
     204            self.logger.debug("Unable to check astrometric solution")
     205
     206        return True
     207
     208    '''
    137209    Have we already imported this DVO table?
    138210    '''
    … …  
    145217            rs.first()
    146218            if rs.getInt(1) > 0:
    147                 self.logger.error("DVO tables " + name + " have already been imported")
     219                self.logger.errorPair("Already imported DVO tables for",  name)
    148220                return True
    149221            else:
    … …  
    158230    def createDvoTables(self):
    159231
    160         self.logger.infoPair("Creating DVO table", "dvoMeta")
     232        self.logger.debugPair("Creating DVO table", "dvoMeta")
    161233
    162234        sql = "DROP TABLE dvoMeta"
    … …  
    174246               flags INT, \
    175247               photcode INT, \
    176                PRIMARY KEY (sourceID, imageID) \
     248               PRIMARY KEY (sourceID, imageID, externID) \
    177249               )"
    178250
    … …  
    181253            self.logger.error("Unable to create DVO meta-data database table")
    182254
    183         self.logger.infoPair("Creating DVO table", "dvoDetection")
     255        self.logger.debugPair("Creating DVO table", "dvoDetection")
    184256        sql = "CREATE TABLE dvoDetection ( \
    185257               sourceID INT, \
    … …  
    192264               PRIMARY KEY (sourceID, imageID, ippDetectID) \
    193265               )"
    194                #INDEX (sourceID), \
    195                #INDEX (imageID), \
    196                #INDEX (ippDetectID) \
    197266
    198267        try: self.execute(sql)
    199268        except:
    200269            self.logger.error("Unable to create DVO detection database table")
     270
     271
     272    '''
     273    Drops and recreates tables necessary for dvoToMySQL program. Be very careful before using this...
     274    '''
     275    def resetDvoToMysqlTables(self):
     276
     277       self.logger.infoPair("Dropping table", self.dvoMetaTable)
     278       self.dropTable(self.dvoMetaTable)
     279       self.logger.infoPair("Dropping table", self.dvoDetectionTable)
     280       self.dropTable(self.dvoDetectionTable)
     281       self.logger.infoPair("Dropping table", self.dvoDoneTable)
     282       self.dropTable(self.dvoDoneTable)
     283
     284       self.logger.infoPair("Creating table", self.dvoMetaTable)
     285       sql = "CREATE TABLE " + self.dvoMetaTable + " LIKE dvoMeta"
     286       try: self.execute(sql)
     287       except: self.logger.errorPair("Unable to create table", self.dvoMetaTable)
     288
     289       self.logger.infoPair("Creating table", self.dvoDetectionTable)
     290       sql = "CREATE TABLE " + self.dvoDetectionTable + " LIKE dvoDetection"
     291       try: self.execute(sql)
     292       except: self.logger.errorPair("Unable to create table", self.dvoDetectionTable)
     293
     294       self.logger.infoPair("Creating table", self.dvoDoneTable )
     295       sql = "CREATE TABLE " + self.dvoDoneTable + " (name VARCHAR(100))"
     296       try: self.execute(sql)
     297       except: self.logger.errorPair("Unable to create table", self.dvoDoneTable)
     298
     299
     300

  • branches/eam_branches/ipp-20110710/ippToPsps/jython/stackbatch.py

    r31847 r32337  
    2727    def __init__(self,
    2828                 logger,
     29                 configPath,
    2930                 configDoc,
    3031                 gpc1Db,
    … …  
    3435       super(StackBatch, self).__init__(
    3536               logger,
     37               configPath,
    3638               configDoc,
    3739               gpc1Db,
    3840               ippToPspsDb,
    3941               stackID,
    40                "stack",
    41                gpc1Db.getStackStageCmf(stackID),
    42                "3PI") # TODO
     42               "ST",
     43               gpc1Db.getStackStageCmf(stackID))
    4344
    4445       if not self.everythingOK: return
    … …  
    4849       # get stack meta data
    4950       meta = self.gpc1Db.getStackStageMeta(self.id)
     51       if not meta:
     52           self.everythingOK = False
     53           return
     54
    5055       self.filter = meta[0];
    5156       self.filter = self.filter[0:1]
    5257       self.skycell = meta[1];
    53        self.skycell = self.skycell[8:]
     58       # TODO HACK fix this
     59       self.skycell = self.skycell[8:12]
    5460       # mangling e.g. 0683.043 into 0683043 for now until we have a schema change
    55        self.skycell = self.skycell.replace(".", "")
     61       #self.skycell = self.skycell.replace(".", "")
    5662       self.analysisVer = meta[2];
    5763
    … …  
    8793       # insert sourceID/imageID combo so DVO can look it up
    8894       if not self.useFullTables:
    89            self.scratchDb.insertNewDvoImage(self.header['SOURCEID'], self.header['IMAGEID'])
    90 
    91     '''
    92     Prints metadata to the log
    93     '''
    94     def printMe(self):
    95 
    96        super(StackBatch, self).printMe()
    97 
     95           self.scratchDb.insertNewDvoExternID(self.header['SOURCEID'], self.header['IMAGEID'])
     96
     97
     98       # dump stuff to log
    9899       self.logger.infoPair("Stack ID", "%d" % self.id)
    99100       self.logger.infoPair("Stack type", "%s" % self.stackType)
    100101       self.logger.infoPair("Skycell", "%s" % self.skycell)
    101102       self.logger.infoPair("Filter", "%s" % self.filter)
     103
    102104
    103105
    … …  
    383385        self.updateStackTypeID("StackDetection")
    384386        self.updateDvoIDs("StackDetection")
    385         sql = "ALTER TABLE StackDetection ADD PRIMARY KEY (objID, stackDetectID)"
    386         self.scratchDb.execute(sql)
    387 
     387        sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)"
     388        self.scratchDb.execute(sql)
     389 
    388390        if self.stackType == "DEEP_STACK":
    389391
    … …  
    392394                   SET instFlux = 2*b.PSF_INST_FLUX_SIG \
    393395                   WHERE instFlux IS NULL \
     396                   AND a.ippDetectID = b.IPP_IDET  \
    394397                   AND b.PSF_INST_FLUX_SIG IS NOT NULL"
    395398            self.scratchDb.execute(sql)
    396             #    instFlux = 2*PSF_INST_FLUX_SIG
    397399           
    398400        self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux")
    … …  
    588590
    589591        imageID = self.scratchDb.getImageIDFromExternID(self.header['SOURCEID'], self.header['IMAGEID'])
    590 
    591592        self.logger.debug("Updating table '" + table + "' with DVO IDs...")
    592         sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \
     593        sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \
    593594               a.ippObjID = b.ippObjID, \
    594595               a.stackDetectID = b.detectID, \
Note: See TracChangeset for help on using the changeset viewer.