Changeset 37551

Timestamp:

Nov 3, 2014, 2:41:20 PM (12 years ago)

Author:

heather

Message:

added diff / ff stages to addstar, partially to ipptopsps

Location:

trunk

Files:

• ippScripts/scripts/addstar_run.pl (modified) (4 diffs)
• ippTasks/addstar.pro (modified) (6 diffs)
• ippToPsps/config/ippToPspsDbSchema.sql (modified) (1 diff)
• ippToPsps/jython/Makefile.am (modified) (1 diff)
• ippToPsps/jython/batch.py (modified) (1 diff)
• ippToPsps/jython/detectionbatch.py (modified) (1 diff)
• ippToPsps/jython/dvoforceddetections.py (added)
• ippToPsps/jython/dvoforcedobjects.py (added)
• ippToPsps/jython/forcedobjectbatch.py (modified) (1 diff)
• ippToPsps/jython/forcedwarpbatch.py (modified) (1 diff)
• ippToPsps/jython/gpc1db.py (modified) (2 diffs)
• ippToPsps/jython/ipptopspsdb.py (modified) (1 diff)
• ippTools/share/Makefile.am (modified) (5 diffs)
• ippTools/share/addtool_find_diff_id.sql (added)
• ippTools/share/addtool_find_diff_id_dvo.sql (added)
• ippTools/share/addtool_find_ff_id.sql (added)
• ippTools/share/addtool_find_ff_id_dvo.sql (added)
• ippTools/share/addtool_find_pendingexp_diff.sql (added)
• ippTools/share/addtool_find_pendingexp_ff.sql (added)
• ippTools/share/addtool_find_processedexp_diff.sql (added)
• ippTools/share/addtool_find_processedexp_ff.sql (added)
• ippTools/share/addtool_queue_diff_id.sql (added)
• ippTools/share/addtool_queue_ff_id.sql (added)
• ippTools/share/addtool_revertprocessedexp_diff.sql (added)
• ippTools/share/addtool_revertprocessedexp_ff.sql (added)
• ippTools/src/addtool.c (modified) (13 diffs)
• ippTools/src/addtoolConfig.c (modified) (4 diffs)
• ippTools/src/pxadd.c (modified) (1 diff)

trunk/ippScripts/scripts/addstar_run.pl

@@ -94 +94 @@
 # Recipes to use based on reduction class
 $reduction = 'DEFAULT' unless defined $reduction;
-if ($stage =~/skycal/) {
-    $reduction = 'DEFAULT';
-}
+if ($stage =~/diff/) {
+    $reduction = 'ADDSTAR';
+} 
+if ($stage =~/forcedwarp/) {
+    $reduction = 'ADDSTAR';
+} 
+
 my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR');
 # XXX This is now not used: do we still need it?
@@ -132 +136 @@
     print "using $fpaObjects for $stage\n";
 }
+if ($stage =~/diff/) {
+    print "using $fpaObjects for $stage\n";
+}
+if ($stage =~/forcedwarp/)  {
+    print "using $fpaObjects for $stage\n";
+} 
+
 
 
@@ -177 +188 @@
 }
 
-if (($stage =~/stack/ || $stage =~ /skycal/)) {
+if (($stage =~/stack/ || $stage =~ /skycal/ || $stage =~/diff/ || $stage=~/forcedwarp/)) {
     $fpaObjects =~ s/smf$/cmf/;
     $fpaObjectsAlt =~ s/smf$/cmf/;
@@ -305 +316 @@
                 $command .= " -quick-airmass";
             }  #careful here - this matches staticsky and staticsky_multi
+            if ($stage =~ /diff/) {
+                $command .= " -accept-astrom";
+            }
+            if ($stage =~ /forcedwarp/) {
+                $command .= " -accept-astrom -xrad"; 
+            }
 
             my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script

trunk/ippTasks/addstar.pro

@@ -52 +52 @@
     active false
   end
+  task addstar.revert.diff
+    active false
+  end
+  task addstar.revert.fullforce
+    active false
+  end
 end
 
@@ -65 +71 @@
   end
   task addstar.revert.skycal
+    active true
+  end
+  task addstar.revert.diff
+    active true
+  end
+  task addstar.revert.fullforce
     active true
   end
@@ -141 +153 @@
 $addstar_revert_DB_SSM = 0
 $addstar_revert_DB_SC = 0 
+$addstar_revert_DB_FF = 0 
+$addstar_revert_DB_DF = 0 
 
 #$LOADEXEC_ADD = 300
@@ -275 +289 @@
         sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID
     end
+    if ("$STAGE" == "diff")
+        sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID
+    end
+    if ("$STAGE" == "fullforce")
+        sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID
+    end
 
   
@@ -293 +313 @@
    if ("$STAGE" == "skycal")
       $run = $run --stage_id $STAGE_ID
+    end
+   if ("$STAGE" == "diff")
+      $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1
+    end
+   if ("$STAGE" == "fullforce")
+      $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1
     end
 
@@ -526 +552 @@
   end
 end
+
+
+task addstar.revert.diff
+  host         local
+
+  periods      -poll 5.0
+  periods      -exec 60.0
+  periods      -timeout 1200
+  npending     1
+  active        false
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = addtool -revertprocessedexp -stage diff
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks                                                                   
+      option $DB:$addstar_revert_DB_DF
+      $run = $run -dbname $DB:$addstar_revert_DB_DF
+      $addstar_revert_DB_DF ++
+      if ($addstar_revert_DB_DF >= $DB:n) set addstar_revert_DB_DF = 0
+    end
+     add_poll_labels run
+    command $run
+  end
+
+  # success                                                                                                   
+  task.exit    0
+  end
+
+  # locked list                                                                                               
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?                                                                                      
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+
+task addstar.revert.fullforce
+  host         local
+
+  periods      -poll 5.0
+  periods      -exec 60.0
+  periods      -timeout 1200
+  npending     1
+  active        false
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = addtool -revertprocessedexp -stage fullforce
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks                                                                   
+      option $DB:$addstar_revert_DB_FF
+      $run = $run -dbname $DB:$addstar_revert_DB_FF
+      $addstar_revert_DB_FF ++
+      if ($addstar_revert_DB_FF >= $DB:n) set addstar_revert_DB_FF = 0
+    end
+     add_poll_labels run
+    command $run
+  end
+
+  # success                                                                                                   
+  task.exit    0
+  end
+
+  # locked list                                                                                               
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?                                                                                      
+  task.exit    timeout
+    showcommand timeout
+  end
+end

trunk/ippToPsps/config/ippToPspsDbSchema.sql

@@ -189 +189 @@
 SET character_set_client = @saved_cs_client;
 --
+-- Table structure for table `forcedobject`
+--
+
+DROP TABLE IF EXISTS `forcedobject`;
+SET @saved_cs_client     = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `forcedobject` (
+  `batch_id` bigint(20) unsigned NOT NULL,
+  `region` varchar(50) default NULL,
+  KEY `batch_id` (`batch_id`),
+  CONSTRAINT `diffobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+SET character_set_client = @saved_cs_client;
+
+--
 -- Table structure for table `diffobject`
 --

trunk/ippToPsps/jython/Makefile.am

@@ -14 +14 @@
   diffobjectbatch.py \
   dvodetections.py \
+  dvoforceddetections.py \
+  dvoforcedobjects.py \
   dvoobjects.py \
   dvo.py \

trunk/ippToPsps/jython/batch.py

@@ -579 +579 @@
         # for OB, this imports object data from DVO
         if not self.importIppTables():
-            if self.batchType == "OB":
+            if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO"):
                 self.logger.errorPair("skipping this batch", "no dvo tables for this region")
                 self.ippToPspsDb.updateProcessed(self.batchID, 1)

trunk/ippToPsps/jython/detectionbatch.py

@@ -214 +214 @@
         sqlLine.group("completMag",       self.getKeyFloat(header, "%.8f", 'FLIMIT'))
         sqlLine.group("astroScat",        self.getKeyFloat(header, "%.8f", 'CERROR'))
-        sqlLine.group("photoScat",        self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS'))
+        sqlLine.group("photoScat",        self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR'))
         sqlLine.group("numAstroRef",      self.getKeyValue(header, 'NASTRO'))
         sqlLine.group("numPhotoRef",      self.getKeyValue(header, 'NASTRO'))

trunk/ippToPsps/jython/forcedobjectbatch.py

@@ +1 @@
+#!/usr/bin/env jython
+
+import os.path
+import sys
+
+import stilts
+from java.lang import *
+from java.sql import *
+
+from xml.etree.ElementTree import ElementTree, Element, tostring
+
+from batch import Batch
+from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
+from dvoforcedobjects import DvoForcedObjects # need one specific for cpy ?
+from sqlUtility import sqlUtility
+
+import logging.config
+
+'''
+ForcedObjectBatch class
+
+This class, a sub-class of Batch, processes individual DVO region file pairs (cpt, cpy, and cps files). The cpt file contains a list of all objects in that region, the cps file contains the magnitudes and so has a row per filter per object (so, generally, has rows = 5 x number of objects). The Photcode.dat table us used to figure out the number and order of filters as they appear in the cps file.
+
+'''
+class ForcedMeanObjectBatch(Batch):
+
+    '''
+    Constructor
+    '''
+    def __init__(self, 
+                 logger, 
+                 config,
+                 skychunk,
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 dvoID,
+                 batchID):
+
+       super(ForcedMeanObjectBatch, self).__init__(
+               logger,
+               config,
+               skychunk,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               dvoID,
+               batchID,
+               "FO", 
+               None)
+
+       try:
+           self.dvoForcedObjects = DvoForcedObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
+       except:
+           self.logger.errorPair("Unable to create instance of", "DvoForcedObjects")
+           raise
+
+       # create an output filename, which is {dvoINDEX}.FITS
+       self.outputFitsFile = "%08d.FITS" % self.id
+       self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
+
+       # dump stuff to log
+       self.logger.infoPair("DVO INDEX", "%d" % self.id)
+
+    '''
+    Overriden from batch base-class. We import from DVO directly for forced mean objects
+    '''
+    def importIppTables(self, columns="*", filter=""):
+
+        self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id)
+        self.ippToPspsDb.insertForcedMeanObjectMeta(self.batchID, self.region)
+        self.dvoForcedObjects.nativeIngestRegion(self.region)
+
+        cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
+
+        if not self.scratchDb.tableExists(cptTableName):
+            self.logger.infoPair("can't find in scratch db:","cpt table")
+            return False
+
+        cpyTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpy")
+
+        if not self.scratchDb.tableExists(cpyTableName):
+            self.logger.infoPair("can't find in scratch db:","cpy table")
+            return False
+
+        return True
+
+    '''
+    Applies indexes to the PSPS tables
+    '''
+    def alterPspsTables(self):
+
+        return True
+
+    '''
+    Applies indexes to the IPP tables
+    '''
+    def indexIppTables(self):
+
+        # since dvopsps is now used, no action is needed here
+        # self.logger.infoPair("Creating indexes on", "IPP tables")
+
+        return True
+
+    '''
+    Inserts stuff for all mags
+    '''
+    def updateForcedMeanObjectFromCps(self, cpsTable):
+
+        # list of all filters PSPS is interested in
+        # XXX EAM : 2014.07.24 : this list should probably be in a config file somewhere
+        interestedFilters = ['g', 'r', 'i', 'z', 'y']
+        
+        filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters)
+    
+        # get a count of the available filters
+        filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable()
+
+        self.logger.infoPair("Available filters in Photcodes", filters)
+
+        # the 'code' now defines the order in the cps file that the mags are listed for a given filter
+        self.logger.infoPair("Adding magnitudes from", "cps table")
+        for filter in filters:
+
+            filterID = self.scratchDb.getFilterID(filter[1])
+
+            # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000)
+            # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags
+            # XXX EAM : 20140724 this manipulation is no longer needed : [grizy]Flags is now 4 byte (was 8 byte!)
+
+            # set the MeanObject fields based largely on dvopsps cps fields:
+
+            # XXX EAM 20140724 : filterCount is meant to match
+            # Nsecfilt, but is potentially not determined correctly.
+            # use a call to a dvo-native command which knows how to
+            # find Nsecfilt (or save in the db with dvopsps)
+
+            # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting?
+            # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter 
+            #  " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+
+            sql = "UPDATE ForcedMeanObject JOIN \
+                   " + cpsTable + " AS cps ON (cps.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+                   SET \
+                    ForcedMeanObject." + filter[1] + "nTotal         = NWARP \
+                   ,ForcedMeanObject." + filter[1] + "nIncPSFMag     = NUSED_WRP \
+                   ,ForcedMeanObject." + filter[1] + "nIncKronMag    = NUSED_KRON_WRP \
+                   ,ForcedMeanObject." + filter[1] + "nIncApMag      = NUSED_AP_WRP \
+                   ,ForcedMeanObject." + filter[1] + "PSFMag         = MAG_PSF_WRP \
+                   ,ForcedMeanObject." + filter[1] + "PSFMagErr      = MAG_PSF_WRP_ERR \
+                   ,ForcedMeanObject." + filter[1] + "PSFMagStd      = MAG_PSF_WRP_STD \
+                   ,ForcedMeanObject." + filter[1] + "KronMag        = MAG_KRON_WRP \
+                   ,ForcedMeanObject." + filter[1] + "KronMagErr     = MAG_KRON_WRP_ERR \
+                   ,ForcedMeanObject." + filter[1] + "KronMagStd     = MAG_KRON_WRP_STD \
+                   ,ForcedMeanObject." + filter[1] + "ApMag          = MAG_AP_WRP \
+                   ,ForcedMeanObject." + filter[1] + "ApMagErr       = MAG_AP_WRP_ERR \
+                   ,ForcedMeanObject." + filter[1] + "ApMagStd       = MAG_AP_WRP_STD \
+                   ,ForcedMeanObject." + filter[1] + "Flags          = FLAGS "
+
+
+            try: self.scratchDb.execute(sql)
+            except:
+                self.logger.errorPair("failed update MeanObject", sql)
+                raise
+
+        # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12)
+        # XXX EAM 20140724 : keep this cut?
+        cut = 0.5
+        self.logger.infoPair("Setting to NULL all MeanMagErr value >", "%f" % cut)
+        for filter in filters:
+
+            sql = "UPDATE ForcedMeanObject \
+                   SET " + filter[1] + "PSFMagErr = null \
+                   WHERE " + filter[1] + "PSFMagErr > " + str(cut)
+            self.scratchDb.execute(sql)
+
+        self.logger.infoPair("Calculating nDetections from", "n[filters]")
+        for filter in filters:
+            # now do a sum of n[filters], but do not include the ones with -999
+            sql  = "UPDATE ObjectThin "
+            sql += "SET nDetections = nDetections + n" + filter[1]
+            sql += " WHERE n" + filter[1] + " != -999"
+            self.scratchDb.execute(sql)
+
+    '''
+    Inserts stuff for all mags from the cpy files
+    '''
+    def updateForcedMeanObjectFromCpy(self, cpyTable):
+
+        # list of all filters PSPS is interested in
+        # XXX EAM : 2014.07.24 : this list should probably be in a config file somewhere
+        interestedFilters = ['g', 'r', 'i', 'z', 'y']
+        
+        filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters)
+    
+        # get a count of the available filters
+        filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable()
+
+        self.logger.infoPair("Available filters in Photcodes", filters)
+
+        # the 'code' now defines the order in the cps file that the mags are listed for a given filter
+        self.logger.infoPair("Adding magnitudes from", "cpy table")
+        for filter in filters:
+
+            filterID = self.scratchDb.getFilterID(filter[1])
+
+            # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000)
+            # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags
+            # XXX EAM : 20140724 this manipulation is no longer needed : [grizy]Flags is now 4 byte (was 8 byte!)
+
+            # set the MeanObject fields based largely on dvopsps cps fields:
+
+            # XXX EAM 20140724 : filterCount is meant to match
+            # Nsecfilt, but is potentially not determined correctly.
+            # use a call to a dvo-native command which knows how to
+            # find Nsecfilt (or save in the db with dvopsps)
+
+            # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting?
+            # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter 
+            #  " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+
+            sql = "UPDATE ForcedMeanObject JOIN \
+                   " + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+                   SET \
+                    ForcedMeanObject." + filter[1] + "nIncR5          =  NMEAS \
+                   ,ForcedMeanObject." + filter[1] + "nIncR6          =  NMEAS \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR5      =  FLUX_AP_R5 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR5Err   =  FLUX_ERR_AP_R5 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR5Std   =  FLUX_STD_AP_R5 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR5Fill  =  FLUX_FIL_AP_R5 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR6      =  FLUX_AP_R6 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR6Err   =  FLUX_ERR_AP_R6 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR6Std   =  FLUX_STD_AP_R6 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR6Fill  =  FLUX_FIL_AP_R6 \
+                   ,ForcedMeanObject." + filter[1] + "LensObjSmearX11 =  X11_SM_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjSmearX12 =  X12_SM_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjSmearX22 =  X22_SM_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjSmearE1  =  E1_SM_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjSmearE2  =  E2_SM_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjShearX11 =  X11_SH_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjShearX12 =  X12_SH_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjShearX22 =  X22_SH_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjShearE1  =  E1_SH_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjShearE2  =  E2_SH_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFSmearX11 =  X11_SM_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFSmearX12 =  X12_SM_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFSmearX22 =  X22_SM_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFSmearE1  =  E1_SM_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFSmearE2  =  E2_SM_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFShearX11 =  X11_SH_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFShearX12 =  X12_SH_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFShearX22 =  X22_SH_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFShearE1  =  E1_SH_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFShearE2  =  E2_SH_PSF \
+                   ,ForcedMeanObject." + filter[1] + "Gamma           =  GAMMA \
+                   ,ForcedMeanObject." + filter[1] + "E1              =  E1 \
+                   ,ForcedMeanObject." + filter[1] + "E2              =  E2 "
+
+
+            try: self.scratchDb.execute(sql)
+            except:
+                self.logger.errorPair("failed update on ForcedMeanObject using cpy table", sql)
+                raise
+
+
+    '''
+    Populates the ForcedMeanObject table
+    '''
+    def populateForcedMeanObjectTable(self):
+
+        cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
+        cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps")
+        cpyTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpy")
+
+                
+        self.logger.info("Populating ForcedMeanObject")
+        self.logger.info("Inserting objects from cpt file")
+
+        # note "dec" is a reserved word in MySQL
+        # XXX EAM 20140724 : do not use INGORE unless we discover unavoidable problems...
+        # INSERT IGNORE INTO ObjectThin
+
+        sqlLine = sqlUtility("INSERT INTO ForcedMeanObject (")
+
+        sqlLine.group("objID",           "EXT_ID") 
+        sqlLine.group("ippObjID",        "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly
+        sqlLine.group("batchID",         "'" + str(self.batchID) + "'")
+        sqlLine.group("nDetections",      "'0'")
+        sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName)
+
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair("Couldn't populate Object table from cpt", sql)
+            return False
+ 
+        # add row count columns so we can perform joins to get colors
+        self.logger.infoPair("Adding 'row' columns to", "Object and cps tables")
+        self.scratchDb.addRowCountColumn("ForcedMeanObject", "row")
+        self.scratchDb.addRowCountColumn(cpsTableName, "row")
+
+
+        self.logger.infoPair("update ForcedMeanObject from ","cps table")
+
+        self.updateForcedMeanObjectFromCps(cpsTableName)
+
+        self.logger.infoPair("update ForcedMeanObject from ","cpy table")
+
+        self.updateForcedMeanObjectFromCpy(cpyTableName)
+
+
+        self.logger.infoPair("Dropping row column from", "ForcedMeanObject table")
+        self.scratchDb.dropColumn("ForcedMeanObject", "row")
+
+        ##self.logger.infoPair("Purging from scratch Db", self.region + " region")
+
+        ##Don't do this till after MeanObject
+        ##self.dvoForcedObjects.purgeRegion(self.region)
+
+        self.setMinMaxObjID(["ForcedMeanObject"])
+
+        return True
+
+ 
+    '''
+    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+    '''
+    def populatePspsTables(self):
+
+        if not self.populateForcedMeanObjectTable(): return False
+
+        # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number
+        self.logger.infoPair("Forcing uniqueness on", "objID in ForcedMeanObject table")
+        rowCountBefore = self.scratchDb.getRowCount("ForcedMeanObject")
+
+        # XXX EAM : note that in mysql versions later than 5.1, this fails
+        # unless the following is called first: 
+        # set session old_alter_table=1
+        # follow the command with 
+        # set session old_alter_table=0
+        # OF COURSE, this fails for mysql version < 5.5...
+        if self.scratchDb.version > 5.1:
+            self.scratchDb.execute("set session old_alter_table=1")
+            
+        self.scratchDb.execute("ALTER IGNORE TABLE ForcedMeanObject ADD UNIQUE INDEX(objID)")
+        if self.scratchDb.version > 5.1:
+            self.scratchDb.execute("set session old_alter_table=0")
+
+        rowCountAfter = self.scratchDb.getRowCount("ForcedMeanObject")
+        self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore))
+
+        self.dvoForcedObjects.purgeRegion(self.region)
+
+        #this is abuse of something but this is how I get the object batches to crash to further investigate them
+        
+#        rowCountAfter = self.scratchDb.getRowCount("Object")
+        return True
+#        return False

trunk/ippToPsps/jython/forcedwarpbatch.py

@@ +1 @@
+#!/usr/bin/env jython
+
+import os.path
+import sys
+import glob
+import time
+import stilts
+
+from java.lang import *
+from java.sql import *
+
+from java.lang import Math
+from org.apache.commons.math.special import Erf
+
+from xml.etree.ElementTree import ElementTree, Element, tostring
+
+from batch import Batch
+from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
+from sqlUtility import sqlUtility
+
+import logging.config
+
+
+'''
+ForcedWarpBatch class
+
+This class, a sub-class of Batch, processes single exposures in the form of IPP smf files
+
+'''
+class ForcedWarpBatch(Batch):
+
+    '''
+    Constructor
+    '''
+    def __init__(self, 
+                 logger,
+                 config,
+                 skychunk,
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 forcedWarpID,
+                 batchID):
+
+       super(ForcedWarpBatch, self).__init__(
+               logger,
+               config,
+               skychunk,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               forcedWarpID,
+               batchID,
+               "P2", 
+               gpc1Db.getForcedWarpStageSmf(forcedWarpID)
+
+       # get camera meta data
+       meta = self.gpc1Db.getForcedWarpStageMeta(self.id)
+       if not meta:
+           self.logger.errorPair("Could not get", "forced warp metadata")
+           raise
+
+       self.expID = meta[0];
+       self.expName = meta[1];
+       self.distGroup = meta[2];
+       self.analysisVer = meta[3];
+       self.bias = meta[4];
+       self.biasScat = meta[5];
+
+       if not self.analysisVer: self.analysisVer = -999
+
+       # create an output filename, which is {expID}.FITS
+       self.outputFitsFile = "%08d.FITS" % self.expID
+       self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
+       self.dropTableVerbose("ForcedWarpMeasurement")
+       self.dropTableVerbose("ForcedWarpExtended")
+       self.dropTableVerbose("ForcedWarpMeta")
+       self.dropTableVerbose("ForcedWarpToImage")
+       if not self.config.retry:
+       self.scratchDb.dropTable("ForcedWarpMeasurement")
+       self.scratchDb.dropTable("ForcedWarpExtended")
+       self.scratchDb.dropTable("ForcedWarpMeta")
+       self.scratchDb.dropTable("ForcedWarpToImage")
+       self.historyModNum = "0"
+
+       # get a few primary header values. if in test mode, then use defaults
+       if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL":
+           self.logger.errorPair("Could not get", "MJD-OBS")
+           raise
+
+
+       if self.safeDictionaryAccessWithDefault(self.header, 'EXPTIME', "1") == "NULL":
+           self.logger.errorPair("Could not get", "EXPTIME")
+           raise
+
+       if self.safeDictionaryAccessWithDefault(self.header, 'FILTERID', "g.0000") == "NULL":
+           self.logger.errorPair("Could not get", "FILTERID")
+           raise
+       
+       # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2
+       self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0)
+      
+       # set up some defauts
+       self.calibModNum = 0
+       self.totalNumPhotoRef = 0
+
+
+       self.filter = self.header['FILTERID'][0:1]
+       self.filterID = self.scratchDb.getFilterID(self.filter) 
+
+       # insert what we know about this stack batch into the stack table
+       self.ippToPspsDb.insertDetectionMeta(self.batchID, self.expID, self.filter)
+
+       # dump stuff to log
+       self.logger.infoPair("Cam ID", "%d" % self.id)
+       self.logger.infoPair("Exp ID", "%d" % self.expID)
+       self.logger.infoPair("Exp name", "%s" % self.expName)
+       self.logger.infoPair("Distribution group", "%s" % self.distGroup)
+
+
+
+
+    '''
+    Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header
+    '''
+    def populateForcedWarpMeta(self):
+        self.logger.infoPair("Processing table", "ForcedWarpMeta")
+
+        photoCalID = str(self.scratchDb.getPhotoCalID(forcedWarpID))       
+
+
+        sqlLine = sqlUtility("INSERT INTO ForcedWarpMeta (")
+        #batchID (below)
+        #surveyID (below)
+        #filterdID (below)
+        sqlLine.group("skyCellID",     str(self.skycellID)) 
+        sqlLine.group("photoCalID",    photoCalID) 
+        sqlLine.group("magSat",           self.getKeyFloat(header, "%.8f", 'FSATUR'))
+        sqlLine.group("analVer",           self.analysisVer);
+        sqlLine.group("expTime",               self.getKeyFloat(self.header, "%.8f", 'EXPREQ'));
+        sqlLine.group("completMag",       self.getKeyFloat(header, "%.8f", 'FLIMIT'))
+        sqlLine.group("astroScat",        self.getKeyFloat(header, "%.8f", 'CERROR'))
+        sqlLine.group("photoScat",             self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR'));
+        sqlLine.group("nAstroRef",      self.getKeyValue(header, 'NASTRO'))
+        sqlLine.group("nPhotoRef",      self.getKeyValue(header, 'NASTRO'))
+        sqlLine.group("psfModelID",     self.getKeyValue(header, 'PSFMODEL'))
+        sqlLine.group("psfFwhm_mean",   self.getKeyFloat(header, "%.8f", 'FWHM_MAJ'))
+        sqlLine.group("psfFwhm_max",    self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ'))
+        sqlLine.group("photoZero",        self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS'))
+        #photoColor -- how do I set this? it's also not set in stack meta
+        sqlLine.group("ctype1",                self.getKeyValue(self.header, 'CTYPE1'));
+        sqlLine.group("ctype2",                self.getKeyValue(self.header, 'CTYPE2'));
+        sqlLine.group("crval1",                self.getKeyFloat(self.header, "%.8f", 'CRVAL1'));
+        sqlLine.group("crval2",                self.getKeyFloat(self.header, "%.8f", 'CRVAL2'));
+        sqlLine.group("crpix1",                self.getKeyFloat(self.header, "%.8f", 'CRPIX1'));
+        sqlLine.group("crpix2",                self.getKeyFloat(self.header, "%.8f", 'CRPIX2'));
+        sqlLine.group("cdelt1",                self.getKeyFloat(self.header, "%.8e", 'CDELT1'));
+        sqlLine.group("cdelt2",                self.getKeyFloat(self.header, "%.8e", 'CDELT2'));
+        sqlLine.group("pc001001",              self.getKeyFloat(self.header, "%.8e", 'PC001001'));
+        sqlLine.group("pc001002",              self.getKeyFloat(self.header, "%.8e", 'PC001002'));
+        sqlLine.group("pc002001",              self.getKeyFloat(self.header, "%.8e", 'PC002001'));
+        sqlLine.group("pc002002",              self.getKeyFloat(self.header, "%.8e", 'PC002002'));
+
+
+        sql = sqlLine.make(") VALUES ( ", ")")
+
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair('failed sql: ', sql)
+            raise
+
+        self.scratchDb.updateAllRows("ForcedWarpMeta", "batchID", str(self.batchID))
+        self.scratchDb.updateAllRows("ForcedWarpMeta", "surveyID", str(self.surveyID))
+        self.scratchDb.updateFilterID("ForcedWarpMeta", self.filter)
+        self.scratchDb.updateAllRows("ForcedWarpMeta", "calibModNum", str(self.calibModNum))
+        self.scratchDb.updateAllRows("ForcedWarpMeta", "dataRelease", str(self.skychunk.dataRelease))
+        self.tablesToExport.append("ForcedWarpMeta")
+       
+    '''
+    Populates the ForcedWarpToImage table
+    '''
+    def populateForcedWarpToImage(self):
+
+        self.logger.infoPair("Procesing table", "ForcedWarpToImage")
+
+        for filter in self.filters:
+            
+            forcedwarpID = self.forcedwarpID
+            if forcedwarpID > 0:
+    
+                imageIDs = self.gpc1Db.getImageIDsForThisForcedWarpID(forcedwarpID)
+
+                for imageID in imageIDs:
+
+                    sql = "INSERT INTO ForcedWarpToImage (forcedwarpID, imageID) \
+                   VALUES (\
+                   " + str(forcedwarpID) + ", " + imageID + ")"
+                    self.scratchDb.execute(sql)
+
+                    # now update StackMeta with correct number of inputs
+
+        self.tablesToExport.append("ForcedWarpToImage")
+
+
+
+
+
+    '''
+    Populates the ForcedWarpMeasurement table
+    '''
+    def populateForcedWarpMeasurementTable(self, results):
+
+        pspsTableName = "ForcedWarpMeasurement"
+        ippTableName = "ForcedWarpMeasurement_psf"
+
+
+        self.logger.infoPair("Procesing table", "ForcedWarpMeasurement")
+
+        # insert the per-object data (IDs, random stack ID, dataRelease, etc)
+        self.selectDvoObjIDs()
+
+        self.generateRandomIDs()
+
+        # add indexes ForcedWarpMeasurement
+        self.scratchDb.createIndex("ForcedWarpMeasurement", "objID")
+
+               
+
+
+
+
+
+
+
+        self.tablesToExport.append("ForcedWarpMeasurement")
+
+
+    '''
+    Populates the ForcedWarpExtended table
+    '''
+    def populateForcedWarpExtendedTable(self, results):
+
+        pspsTableName = "ForcedWarpExtended"
+        ippTableName = "ForcedWarpExtended_psf"
+
+    
+        self.tablesToExport.append("ForcedWarpExtended")
+
+
+    '''
+    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+    '''
+    def populatePspsTables(self):
+        self.logger.infoPair("starting","populatePspsTables");
+        self.skipBatch = False
+        # each of the "populate*" methods below add their tables to the list:
+        self.tablesToExport=[]
+    
+        #this should be done
+        self.populateForcedWarpMeta()          
+
+        self.populateForcedWarpMeasurement()           
+        self.populateForcedWarpExtended()
+        self.populateForcedWarpToImage()
+             
+        self.setMinMaxObjID(["ForcedWarpMeasurement"])
+
+        self.logger.infoPair("finishing","populatePspsTables"); 
+        return True
+          
+
+
+    '''
+    This function reads the cmf/smf file and loads it into the database as its own table
+    '''
+    def importIppTables(self, columns="*", tableRE=""):
+        
+        if self.config.retry: return True
+        self.logger.infoPair("Importing FW tables with table match expression: ", tableRE)
+        fileName = self.fits.getPath()
+
+        self.logger.infoPair("using filename:",fileName)
+            
+        try:
+           tables = stilts.treads(fileName)
+        except:
+           self.logger.errorPair("STILTS could not import from", fileName)
+           return False
+        for table in tables:
+              
+           match = re.match(tableRE, table.name)
+           if not match: continue
+           self.logger.infoPair("Reading IPP table", table.name)
+           table = stilts.tpipe(table, cmd='addcol table_index $0')
+           table = stilts.tpipe(table, cmd='explodeall')
+           
+           # drop any previous tables before import
+           self.scratchDb.dropTable(table.name)
+                  
+           # IPP FITS files are littered with infinities, so remove these
+           self.logger.info("Removing Infinity values from all columns")
+           table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
+           table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
+           table = stilts.tpipe(table, cmd='replaceval Infinity null *')
+                  
+           try:
+               table.write(self.scratchDb.url + '#' + table.name)
+               count = count + 1
+           except:
+               self.logger.exception("Problem writing table '" + table.name + "' to the database")
+                      
+        self.logger.infoPair("Done. Imported", "%d tables" % count)
+        self.indexIppTables()
+                      
+        return True
+
+
+               
+
+
+
+
+
+
+    def generateRandomIDs(self):
+        sql = "UPDATE ForcedWarpMeasurement set randomWarpID = FLOOR(RAND()*9223372036854775807)";
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair('failed sql',sql)
+            return
+
+
+
+
+
+
+
+
+
+    def exportPspsTablesToFits(self, regex="(.*)"):
+       return super(DetectionBatch, self).exportPspsTablesToFits("(ForcedWarp.*)")

trunk/ippToPsps/jython/gpc1db.py

@@ -116 +116 @@
             
             stage = "forcedwarp"
-            sql = "SELECT differentStuff from Stufftable"
+            sql = "SELECT DISTINCT sky_id, radeg, decdeg FROM fullForceInput \
+                   JOIN addRun ON(fullForceInput.ff_id = addRun.stage_id, fullForceInput.warp_id = addRun.stage_extra1) \
+                   JOIN minidvodbRun USING(minidvodb_name) \
+                   JOIN minidvodbProcessed USING(minidvodb_id) \
+                   JOIN mergedvodbRun USING(minidvodb_id) \
+                   JOIN mergedvodbProcessed USING (merge_id) \
+                   JOIN fullForceRun USING(ff_id) \
+                   JOIN skycalRun using (skycal_id)
+                   JOIN stackRun USING(stack_id) \
+                   JOIN stackSumSkyfile using (stack_id) \
+                   JOIN skycell USING(skycell_id) \
+                   WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+                   AND minidvodbRun.state = 'merged' \
+                   AND minidvodbProcessed.fault = 0 \
+                   AND mergedvodbRun.state = 'full' \
+                   AND mergedvodbProcessed.fault = 0 \
+                   AND addRun.stage = '" + stage + "' \
+                   AND addRun.state = 'full' \
+                   AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
+                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+                   AND mjd_obs >= (to_days('" + tstart + "')-678941) \
+                   AND mjd_obs <= (to_days('" + tend + "') - 678941) "
 
         try:
@@ -165 +186 @@
 
         return imageIDs
+
+    
+
+
+
 
     '''

trunk/ippToPsps/jython/ipptopspsdb.py

@@ -756 +756 @@
 
         self.execute(sql)
+
+    '''
+    Inserts some forced detection metadata for this batch ID
+    '''
+    def insertForcedWarpMeta(self, batchID, expID, filter):
+
+        sql = "INSERT INTO forcedwarp ( \
+               batch_id \
+               ,exp_id \
+               ,filter \
+               ) VALUES ( \
+               " + str(batchID) + " \
+               ," + str(expID) + " \
+               ,'" + filter + "' \
+               )"
+
+        self.execute(sql)
+
+
+
+    '''
+    Inserts some forcedobject metadata for this batch ID
+    '''
+    def insertForcedObjectMeta(self, batchID, region):
+
+        sql = "INSERT INTO forcedobject ( \
+               batch_id \
+               ,region \
+               ) VALUES ( \
+               " + str(batchID) + " \
+               ,'" + region + "' \
+               )"
+
+        self.execute(sql)
+
+
+
+
+    ''' 
+    Inserts some diff metadata for this batch ID
+    '''
+    def insertDiffMeta(self, batchID, filter, diffType):
+
+        sql = "INSERT INTO stack ( \
+               batch_id \
+               ,filter \
+               ,diff_type \
+               ) VALUES ( \
+               " + str(batchID) + " \
+               ,'" + filter + "' \
+               ,'" + diffType + "')"
+
+        self.execute(sql)
+
+
+
+
+    '''
+    Inserts some diffobject metadata for this batch ID
+    '''
+    def insertDiffObjectMeta(self, batchID, region):
+
+        sql = "INSERT INTO diffobject ( \
+               batch_id \
+               ,region \
+               ) VALUES ( \
+               " + str(batchID) + " \
+               ,'" + region + "' \
+               )"
+
+        self.execute(sql)
+
+
+
+
+
+
     '''
     Resets a batch ready for re-loading to the datastore

trunk/ippTools/share/Makefile.am

@@ -8 +8 @@
         addtool_find_cam_id_dvo.sql \
         addtool_find_cam_id.sql \
+        addtool_find_diff_id.sql \
+        addtool_find_diff_id_dvo.sql \
+        addtool_find_ff_id.sql \
+        addtool_find_ff_id_dvo.sql \
         addtool_find_skycal_id_dvo.sql \
         addtool_find_skycal_id.sql \
@@ -16 +20 @@
         addtool_find_pendingexp.sql \
         addtool_find_pendingexp_cam.sql \
+        addtool_find_pendingexp_diff.sql \
+        addtool_find_pendingexp_ff.sql \
         addtool_find_pendingexp_skycal.sql \
         addtool_find_pendingexp_stack.sql \
@@ -21 +27 @@
         addtool_find_pendingmergeprocess.sql \
         addtool_find_processedexp_cam.sql \
+        addtool_find_processedexp_diff.sql \
+        addtool_find_processedexp_ff.sql \
         addtool_find_processedexp_skycal.sql \
         addtool_find_processedexp_stack.sql \
@@ -29 +37 @@
         addtool_pendingcleanuprun.sql \
         addtool_queue_cam_id.sql \
+        addtool_queue_diff_id.sql \
+        addtool_queue_ff_id.sql \
         addtool_queue_stack_id.sql \
         addtool_queue_sky_id_multi.sql \
@@ -35 +45 @@
         addtool_revertminidvodbprocessed.sql \
         addtool_revertprocessedexp_cam.sql \
+        addtool_revertprocessedexp_diff.sql \
+        addtool_revertprocessedexp_ff.sql \
         addtool_revertprocessedexp_skycal.sql \
         addtool_revertprocessedexp_stack.sql \

trunk/ippTools/src/addtool.c

@@ -122 +122 @@
     PXOPT_COPY_S64(config->args, where,  "-sky_id",    "staticskyRun.sky_id", "==");
     PXOPT_COPY_S64(config->args, where,  "-skycal_id",  "skycalRun.skycal_id", "==");
+    PXOPT_COPY_S64(config->args, where,  "-diff_id",  "diffRun.diff_id", "==");
+    PXOPT_COPY_S64(config->args, where,  "-ff_id",  "fullForceRun.ff_id", "==");
+
     PXOPT_LOOKUP_STR(stage,       config->args, "-stage", false, false);
     if (strcmp(stage, "cam")== 0) {
@@ -146 +149 @@
       PXOPT_COPY_STR(config->args, where,  "-reduction", "skycalRun.reduction",  "==");
     }
-
+    if (strcmp(stage, "diff")==0) {
+      pxAddLabelSearchArgs (config, where, "-label",     "diffRun.label", "=="); //define using skycalRun label
+      pxAddLabelSearchArgs (config, where, "-data_group","diffRun.data_group", "==");
+      PXOPT_COPY_STR(config->args, where,  "-reduction", "diffRun.reduction",  "==");
+    }
+    if (strcmp(stage, "fullforce")==0) {
+      pxAddLabelSearchArgs (config, where, "-label",     "fullForceRun.label", "=="); //define using skycalRun label
+      pxAddLabelSearchArgs (config, where, "-data_group","fullForceRun.data_group", "==");
+      PXOPT_COPY_STR(config->args, where,  "-reduction", "fullForceRun.reduction",  "==");
+    }
+
+ 
+ 
     if (!psListLength(where->list)) {
         psFree(where);
@@ -258 +273 @@
     }
 
-
+    if (strcmp(stage,"diff") == 0) {
+      if (dvodb ) {
+        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_diff_id_dvo.sql\n%s\n", dvodb,stage);
+        // find the skycal_id of all the exposures that we want to queue up.                                                                                                   
+        bare_query = pxDataGet("addtool_find_diff_id_dvo.sql");
+        // user supplied dvodb                                                                                                                                              
+        psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb);
+      } else {
+        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_diff_id.sql\n%s\n",stage);
+        // find the skycal_id of all the exposures that we want to queue up.                                                                                                   
+        bare_query = pxDataGet("addtool_find_diff_id.sql");
+        // inherit dvodb from skycalRun, avoid matching NULL                                                                                                                   
+        psStringAppend(&dvodb_string, "(diffRun.dvodb IS NOT NULL AND previous_dvodb = diffRun.dvodb)");
+        // this is silly, there is no dvodb in skycalRun...?
+      }
+    }
+
+    if (strcmp(stage,"fullforce") == 0) {
+      if (dvodb ) {
+        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_ff_id_dvo.sql\n%s\n", dvodb,stage);
+        // find the skycal_id of all the exposures that we want to queue up.                                                                                                   
+        bare_query = pxDataGet("addtool_find_ff_id_dvo.sql");
+        // user supplied dvodb                                                                                                                                              
+        psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb);
+      } else {
+        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_ff_id.sql\n%s\n",stage);
+        // find the skycal_id of all the exposures that we want to queue up.                                                                                                   
+        bare_query = pxDataGet("addtool_find_ff_id.sql");
+        // inherit dvodb from skycalRun, avoid matching NULL                                                                                                                   
+        psStringAppend(&dvodb_string, "(fullForceRun.dvodb IS NOT NULL AND previous_dvodb = fullForceRun.dvodb)");
+        // this is silly, there is no dvodb in skycalRun...?
+      }
+    }
 
 
@@ -324 +371 @@
         psStringAppend(&query, " GROUP BY skycal_id, sky_id, stack_id ");  //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there
       }
+      //needs to be checked HAF xxx
+      if (strcmp(stage,"diff") == 0) {
+        psStringAppend(&query, " GROUP BY diff_id ");  //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there
+      }
+      //needs to be checked HAF xxx
+      if (strcmp(stage,"fullforce") == 0) {
+        psStringAppend(&query, " GROUP BY ff_id, skycal_id ");  //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there
+      }
+
+
+
 
     psTrace("addtool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query);
@@ -476 +534 @@
     }
 
+    if (strcmp(stage,"diff") == 0) {
+      for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        diffRunRow *row = diffRunObjectFromMetadata(md);
+
+        if (!row) {
+          psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun");
+          psFree(output);
+          return false;
+        }
+
+        if (!dvodb) {  // there's no skycalRun.dvodb
+          psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, diff_id %" PRId64, row->label, row->diff_id);
+          psFree(output);
+          return false;
+        }
+        if (!workdir && !row->workdir) {
+          psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, diff_id %" PRId64, row->label, row->diff_id);
+          psFree(output);
+          return false;
+        }
+
+        psFree(row);
+      }
+    }
+
+    if (strcmp(stage,"fullforce") == 0) {
+      for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        fullForceRunRow *row = fullForceRunObjectFromMetadata(md);
+
+        if (!row) {
+          psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun");
+          psFree(output);
+          return false;
+        }
+
+        if (!dvodb) {  // there's no skycalRun.dvodb
+          psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, ff_id %" PRId64, row->label, row->ff_id);
+          psFree(output);
+          return false;
+        }
+        if (!workdir && !row->workdir) {
+          psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, ff_id %" PRId64, row->label, row->ff_id);
+          psFree(output);
+          return false;
+        }
+
+        psFree(row);
+      }
+    }
 
 
@@ -672 +783 @@
       }
    
+    if (strcmp(stage,"diff") == 0) {
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+        psS64 stage_id =0; 
+        
+                  diffRunRow *row = diffRunObjectFromMetadata(md);
+          stage_id = row->diff_id;
+        
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun");
+            psFree(output);
+            return false;
+        }
+
+        // queue the exp
+        if (!pxaddQueueByCamID(config,
+                               stage,
+                               stage_id,
+                               0,
+                               workdir     ? workdir   : row->workdir,
+                               reduction   ? reduction : row->reduction,
+                               label       ? label     : row->label,
+                               data_group  ? data_group : (row->data_group ? row->data_group :  (label ? label : row->label)),
+                               dvodb       ? dvodb     : NULL,
+                               note        ? note      : NULL,
+                               image_only,
+                               minidvodb,
+                               minidvodb_group,
+                               minidvodb_name
+        )) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error sfg");
+            }
+            psError(PS_ERR_UNKNOWN, false,
+                    "failed to trying to queue stage %s %" PRId64,stage, stage_id);
+            psFree(row);
+            psFree(output);
+            return false;
+        }
+        psFree(row);
+    }
+      }
+ 
+    if (strcmp(stage,"fullforce") == 0) {
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+        psS64 stage_id =0; 
+        
+                  fullForceRunRow *row = fullForceRunObjectFromMetadata(md);
+          stage_id = row->ff_id;
+        
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun");
+            psFree(output);
+            return false;
+        }
+
+        // queue the exp
+        if (!pxaddQueueByCamID(config,
+                               stage,
+                               stage_id,
+                               0,
+                               workdir     ? workdir   : row->workdir,
+                               reduction   ? reduction : row->reduction,
+                               label       ? label     : row->label,
+                               data_group  ? data_group : (row->data_group ? row->data_group :  (label ? label : row->label)),
+                               dvodb       ? dvodb     : NULL,
+                               note        ? note      : NULL,
+                               image_only,
+                               minidvodb,
+                               minidvodb_group,
+                               minidvodb_name
+        )) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error sfg");
+            }
+            psError(PS_ERR_UNKNOWN, false,
+                    "failed to trying to queue stage %s %" PRId64,stage, stage_id);
+            psFree(row);
+            psFree(output);
+            return false;
+        }
+        psFree(row);
+    }
+      }
+
+
+
 
 
@@ -695 +894 @@
     PXOPT_COPY_S64(config->args, where, "-sky_id",    "staticskyRun.sky_id", "==");
     PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "==");
+
     PXOPT_LOOKUP_STR(stage,       config->args, "-stage", false, false);
     pxcamGetSearchArgs (config, where); // most search arguments based on camera
@@ -709 +911 @@
     psString query = NULL;
     if (strcmp(stage, "cam")==0) {
-      query = psStringCopy("UPDATE addRun JOIN camRun on cam_id = stage_id JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)");
+      query = psStringCopy("UPDATE addRun JOIN camRun on (cam_id = stage_id and stage = 'cam')  JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)");
     }
     if (strcmp(stage, "stack")==0) {
-      query = psStringCopy("UPDATE addRun JOIN stackRun on stack_id = stage_id");
+      query = psStringCopy("UPDATE addRun JOIN stackRun on (stack_id = stage_id and stage = 'stack')");
     }
     if (strcmp(stage, "staticsky")==0) {
-      query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id");
+      query = psStringCopy("UPDATE addRun JOIN staticskyRun on (sky_id = stage_id and stage = 'staticsky')");
     }
     if (strcmp(stage, "skycal")==0) {
-      query = psStringCopy("UPDATE addRun JOIN skycalRun on skycal_id = stage_id");
+      query = psStringCopy("UPDATE addRun JOIN skycalRun on (skycal_id = stage_id and stage = 'skycal')");
+    }
+    if (strcmp(stage, "diff")==0) {
+      query = psStringCopy("UPDATE addRun JOIN diffInputSkyfile on (diff_id = stage_id and diff_skyfile_id = stage_extra1 and stage = 'diff') JOIN diffSkyfile on (diff_id, skycell_id) JOIN diffRun using (diff_id)");
+    }
+    if (strcmp(stage, "fullforce")==0) {
+      query = psStringCopy("UPDATE addRun JOIN fullForceResult on (ff_id = stage_id and warp_id = stage_extra1 and stage = 'fullforce') JOIN fullForceRun on (ff_id)");
     }
 
@@ -743 +951 @@
     PXOPT_COPY_S64(config->args, where, "-sky_id",    "staticskyRun.sky_id", "==");
     PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-ff_id",    "fullForceRun.ff_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-diff_id",    "diffRun.diff_id", "==");
     PXOPT_LOOKUP_STR(stage,       config->args, "-stage", false, false);
     pxcamGetSearchArgs (config, where);
@@ -764 +974 @@
       query = pxDataGet("addtool_find_pendingexp_skycal.sql");
     }
+    if (strcmp(stage, "diff")==0) {
+      query = pxDataGet("addtool_find_pendingexp_diff.sql");
+    }
+    if (strcmp(stage, "fullforce")==0) {
+      query = pxDataGet("addtool_find_pendingexp_ff.sql");
+    }
 
 
@@ -790 +1006 @@
       psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1");
     }
+    if (strcmp(stage, "diff") == 0) {
+      //this group by is needed to join against all the warps (to get camera)
+      psStringAppend(&query, " GROUP BY %s", "diff_id, stage_extra1");
+    }
+    if (strcmp(stage, "fullforce") == 0) {
+      //this group by is needed to join against all the warps (to get camera)
+      psStringAppend(&query, " GROUP BY %s", "ff_id, stage_extra1");
+    }
+
+
+
     // treat limit == 0 as "no limit"
     if (limit) {
@@ -994 +1221 @@
     } else if (strcmp (stage,"skycal") == 0) {
       query = pxDataGet("addtool_find_processedexp_skycal.sql");
+    } else if (strcmp (stage,"diff") == 0) {
+      query = pxDataGet("addtool_find_processedexp_diff.sql");
+    } else if (strcmp (stage,"fullforce") == 0) {
+      query = pxDataGet("addtool_find_processedexp_ff.sql");
+
     } else {
         psFree(where);
@@ -1112 +1344 @@
     if (strcmp(stage, "skycal") == 0) {
       query = pxDataGet("addtool_revertprocessedexp_skycal.sql");
+    }
+    if (strcmp(stage, "diff") == 0) {
+      query = pxDataGet("addtool_revertprocessedexp_diff.sql");
+    }
+    if (strcmp(stage, "fullforce") == 0) {
+      query = pxDataGet("addtool_revertprocessedexp_ff.sql");
     }
 

trunk/ippTools/src/addtoolConfig.c

@@ -55 +55 @@
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-skycal_id",          0, "search by skycal_id", 0);
     pxcamSetSearchArgs(definebyqueryArgs);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id",            0, "search by diff_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-ff_id",              0, "search by ff_id", 0);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by camRun data_group", NULL);
@@ -83 +85 @@
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sky_id",                 0, "search by sky_id", 0);
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-skycal_id",                 0, "search by skycal_id", 0);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id",                0, "search by diff_id",0);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ff_id",                  0, "search by _ff_id",0);
     pxcamSetSearchArgs(updaterunArgs);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label",                  0, "search by addRun label", NULL);
@@ -101 +105 @@
     psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id",            0, "search by add_id", 0);
     psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id",            0, "search by cam_id", 0);
-    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-stack_id",                 0, "search by stack_id", 0);
-    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-sky_id",                 0, "search by sky_id", 0);
-    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-skycal_id",                 0, "search by skycal_id", 0);
+    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-stack_id",                0, "search by stack_id", 0);
+    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-sky_id",                  0, "search by sky_id", 0);
+    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-skycal_id",               0, "search by skycal_id", 0);
+    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-diff_id",                 0, "search by diff_id",0);
+    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-ff_id",                   0, "search by ff_id",0);
     pxcamSetSearchArgs(pendingexpArgs);
     psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL);
@@ -236 +242 @@
     psMetadata *modes = psMetadataAlloc();
 
-    PXOPT_ADD_MODE("-definebyquery",        "create runs from cam stage",           ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs);
+    PXOPT_ADD_MODE("-definebyquery",        "create runs from cam/skycal/forcedwarp/diff stage",           ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs);
     PXOPT_ADD_MODE("-updaterun",            "change add run properties",            ADDTOOL_MODE_UPDATERUN,      updaterunArgs);
     PXOPT_ADD_MODE("-pendingexp",           "show pending exps",                    ADDTOOL_MODE_PENDINGEXP,    pendingexpArgs);

trunk/ippTools/src/pxadd.c

@@ -164 +164 @@
         psMemSetPersistent(query, true);
       }
+      else if (strcmp(stage,"diff") == 0) {
+        query = pxDataGet("addtool_queue_diff_id.sql");
+        psMemSetPersistent(query, true);
+      }
+      else if (strcmp(stage,"fullforce") == 0) {
+        query = pxDataGet("addtool_queue_ff_id.sql");
+        psMemSetPersistent(query, true);
+      }
+
+
       else 
         {