Index: branches/eam_branches/ipp-20130419/ippToPsps/config/ippToPspsDbSchema.sql
===================================================================
--- branches/eam_branches/ipp-20130419/ippToPsps/config/ippToPspsDbSchema.sql (revision 35543)
+++ branches/eam_branches/ipp-20130419/ippToPsps/config/ippToPspsDbSchema.sql (revision 35554)
@@ -48,5 +48,5 @@
`purged` tinyint(4) NOT NULL default '0',
PRIMARY KEY (`batch_id`)
-) ENGINE=InnoDB AUTO_INCREMENT=328286 DEFAULT CHARSET=latin1;
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1;
SET character_set_client = @saved_cs_client;
@@ -69,5 +69,5 @@
KEY `fk_skychunk` (`skychunk`),
CONSTRAINT `fk_skychunk` FOREIGN KEY (`skychunk`) REFERENCES `skychunk` (`name`) ON DELETE CASCADE
-) ENGINE=InnoDB AUTO_INCREMENT=44588 DEFAULT CHARSET=latin1;
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1;
SET character_set_client = @saved_cs_client;
@@ -91,5 +91,5 @@
PRIMARY KEY (`id`),
UNIQUE KEY `unique_key` (`host`,`pid`)
-) ENGINE=InnoDB AUTO_INCREMENT=1227 DEFAULT CHARSET=latin1;
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1;
SET character_set_client = @saved_cs_client;
Index: branches/eam_branches/ipp-20130419/ippToPsps/config/settings.xml
===================================================================
--- branches/eam_branches/ipp-20130419/ippToPsps/config/settings.xml (revision 35543)
+++ branches/eam_branches/ipp-20130419/ippToPsps/config/settings.xml (revision 35554)
@@ -45,4 +45,137 @@
16
+
+ ipptopsps_scratch
+ ipp045
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp046
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp047
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp048
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp049
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp050
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp051
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp052
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp053
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp054
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp055
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp056
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp057
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp058
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp059
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp060
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp061
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp062
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp063
+ dvo
+ dvo
+ 32
+
ipptopsps_scratch
@@ -52,4 +185,58 @@
32
+
+ ipptopsps_scratch
+ ipp065
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ ipp066
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps_scratch
+ stare00
+ dvo
+ dvo
+ 64
+
+
+ ipptopsps_scratch
+ stare01
+ dvo
+ dvo
+ 64
+
+
+ ipptopsps_scratch
+ stare02
+ dvo
+ dvo
+ 64
+
+
+ ipptopsps_scratch
+ stare03
+ dvo
+ dvo
+ 64
+
+
+ ipptopsps_scratch
+ stare04
+ dvo
+ dvo
+ 64
+
+
+
+
+
+
ipptopsps_scratch
Index: branches/eam_branches/ipp-20130419/ippToPsps/doc/notes.txt
===================================================================
--- branches/eam_branches/ipp-20130419/ippToPsps/doc/notes.txt (revision 35543)
+++ branches/eam_branches/ipp-20130419/ippToPsps/doc/notes.txt (revision 35554)
@@ -1,2 +1,41 @@
+
+2013.04.27
+
+ need to move the conversion between mags and flux into dvopsps.
+
+ * detection table
+ * psfFlux : uses PSF_INST_MAG instead of PSF_INST_FLUX
+ * psfFluxErr : uses PSF_INST_MAG_SIG and PSF_INST_MAG instead of PSF_INST_FLUX_SIG
+ * apFlux : uses AP_MAG instead of AP_FLUX
+ * kronFlux : correctly uses KRON_FLUX
+ * kronFluxErr : correctly uses KRON_FLUX_ERR
+
+ ** psfFlux is forced to 0.0000001 instead of 0.0 -- why?
+
+ * stack detection table (same issues as detection table)
+ * psfFlux : uses PSF_INST_MAG instead of PSF_INST_FLUX
+ * psfFluxErr : uses PSF_INST_MAG_SIG and PSF_INST_MAG instead of PSF_INST_FLUX_SIG
+ * apFlux : uses AP_MAG instead of AP_FLUX
+ * kronFlux : correctly uses KRON_FLUX
+ * kronFluxErr : correctly uses KRON_FLUX_ERR
+
+ * psfFlux is set to 2 sigma if psfFlux is NULL
+
+ * object
+ * Mean PSF Mag : direct from DVO MAG
+ * Mean PSF Mag Err : direct from DVO MAG_ERR
+ * Mean Kron Mag : direct from DVO KRON MAG
+ * Mean Kron Mag Err : direct from DVO KRON MAG_ERR
+
+ * Stack PSF Mag : from DVO stack psf flux
+ * Stack PSF Mag Err : from DVO stack psf flux err
+ * Stack Kron Mag : from DVO stack kron flux
+ * Stack Kron Mag Err : from DVO stack kron flux err
+
+
+
+
+
+old notes:
example of joining ipptopsps and gpc1 tables:
Index: branches/eam_branches/ipp-20130419/ippToPsps/jython/batch.py
===================================================================
--- branches/eam_branches/ipp-20130419/ippToPsps/jython/batch.py (revision 35543)
+++ branches/eam_branches/ipp-20130419/ippToPsps/jython/batch.py (revision 35554)
@@ -65,4 +65,5 @@
self.scratchDb = scratchDb
self.batchType = batchType;
+ self.skipBatch = False; # set to true if there is a non-fatal error
self.pspsVoTableFilePath = self.config.configDir + "tables." + batchType + ".vot"
Index: branches/eam_branches/ipp-20130419/ippToPsps/jython/detectionbatch.py
===================================================================
--- branches/eam_branches/ipp-20130419/ippToPsps/jython/detectionbatch.py (revision 35543)
+++ branches/eam_branches/ipp-20130419/ippToPsps/jython/detectionbatch.py (revision 35554)
@@ -439,6 +439,6 @@
,X_PSF_SIG \
,Y_PSF_SIG \
- ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + " \
- ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + ")) / 1.085736) \
+ ,PSF_INST_FLUX / " + self.header['EXPTIME'] + " \
+ ,PSF_INST_FLUX_SIG / " + self.header['EXPTIME'] + " \
,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + self.header['EXPTIME'] + " \
,PSF_MAJOR \
@@ -450,5 +450,5 @@
,MOMENTS_XY \
,MOMENTS_YY \
- ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + " \
+ ,AP_FLUX / " + self.header['EXPTIME'] + " \
,KRON_FLUX / " + self.header['EXPTIME'] + " \
,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \
@@ -466,11 +466,30 @@
# self.logger.info(sql)
- self.scratchDb.execute(sql)
-
+# these were used above (left over from before PSF_INST_FLUX, PSF_INST_FLUX_SIG were available?
+# ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + "
+# ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + ")) / 1.085736)
+# ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + "
+
+ self.scratchDb.execute(sql)
+
+ # XXX EAM : I removed this old fix for invalid fluxes
# add a instFlux = 0.0 -> 0.000001
- # XXX EAM : why is this done?
- sql="UPDATE " + pspsTableName + " SET psfFlux = 0.0000001 WHERE psfFlux = 0"
- self.scratchDb.execute(sql)
+ # XXX EAM : why is this done?
+ # sql="UPDATE " + pspsTableName + " SET psfFlux = 0.0000001 WHERE psfFlux = 0"
+ # self.scratchDb.execute(sql)
+ # NOTE : Flux limits : in the current PSPS schema, negative fluxes
+ # cause problems for sql queries which work in mags as SQL cannot do
+ # something like (f < 0.0) ? -999 : -2.5*log10(f)
+ # as a result, the negative fluxes here result in floating point errors
+ sql = "UPDATE " + pspsTableName + " SET psfFlux = 1e20 WHERE psfFlux <= 0.0"
+ self.scratchDb.execute(sql)
+
+ sql = "UPDATE " + pspsTableName + " SET apFlux = 1e20 WHERE apFlux <= 0.0"
+ self.scratchDb.execute(sql)
+
+ sql = "UPDATE " + pspsTableName + " SET kronFlux = 1e20 WHERE kronFlux <= 0.0"
+ self.scratchDb.execute(sql)
+
# we don't delete these anymore
results['NULLINSTFLUX'] = 0;
@@ -1016,5 +1035,5 @@
# XXX EAM NOTE : this is fragile : requires PS1_V4
- columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ"
+ columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG AP_FLUX KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ"
return super(DetectionBatch, self).importIppTables(columns, regex)
Index: branches/eam_branches/ipp-20130419/ippToPsps/jython/objectbatch.py
===================================================================
--- branches/eam_branches/ipp-20130419/ippToPsps/jython/objectbatch.py (revision 35543)
+++ branches/eam_branches/ipp-20130419/ippToPsps/jython/objectbatch.py (revision 35554)
@@ -137,12 +137,18 @@
," + filter[1] + "MeanKronMag = MAG_KRON \
," + filter[1] + "MeanKronMagErr = MAG_KRON_ERR \
- ," + filter[1] + "StackPSFMag = -2.5 * log10(FLUX_PSF) + 8.9 \
- ," + filter[1] + "StackPSFMagErr = FLUX_PSF_ERR / FLUX_PSF \
- ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) + 8.9 \
- ," + filter[1] + "StackKronMagErr = FLUX_KRON_ERR / FLUX_KRON \
+ ," + filter[1] + "StackPSFMag = STACK_PSF_MAG \
+ ," + filter[1] + "StackPSFMagErr = STACK_PSF_MAG_ERR \
+ ," + filter[1] + "StackKronMag = STACK_KRON_MAG \
+ ," + filter[1] + "StackKronMagErr = STACK_KRON_MAG_ERR \
," + filter[1] + "20pct = MAG_20/1000 \
," + filter[1] + "80pct = MAG_80/1000 \
," + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) \
," + filter[1] + "StackDetectID = STACK_DETECT_ID"
+
+# old method for setting the Stack (PSF,Kron) Mags:
+# ," + filter[1] + "StackPSFMag = -2.5 * log10(FLUX_PSF) + 8.9 \
+# ," + filter[1] + "StackPSFMagErr = FLUX_PSF_ERR / FLUX_PSF \
+# ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) + 8.9 \
+# ," + filter[1] + "StackKronMagErr = FLUX_KRON_ERR / FLUX_KRON \
#self.logger.info(sql)
@@ -216,24 +222,22 @@
cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps")
- self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON_ERR")
- sql = "UPDATE " + cpsTableName + " set FLUX_KRON_ERR = NULL where FLUX_KRON_ERR < 1e-37 AND FLUX_KRON_ERR > -1e-37 "
+ if False:
+ self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON_ERR")
+ sql = "UPDATE " + cpsTableName + " set FLUX_KRON_ERR = NULL where FLUX_KRON_ERR < 1e-37 AND FLUX_KRON_ERR > -1e-37 "
- try:
- self.scratchDb.execute(sql)
- except:
- self.logger.errorPair("Couldn't squash out of range stuff", sql)
- return False
-
- self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON")
- sql = "UPDATE " + cpsTableName + " set FLUX_KRON = NULL where FLUX_KRON < 1e-37 AND FLUX_KRON > -1e-37 "
+ try:
+ self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("Couldn't squash out of range stuff", sql)
+ return False
+
+ self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON")
+ sql = "UPDATE " + cpsTableName + " set FLUX_KRON = NULL where FLUX_KRON < 1e-37 AND FLUX_KRON > -1e-37 "
- try:
- self.scratchDb.execute(sql)
- except:
- self.logger.errorPair("Couldn't squash out of range stuff", sql)
- return False
-
-
-
+ try:
+ self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("Couldn't squash out of range stuff", sql)
+ return False
Index: branches/eam_branches/ipp-20130419/ippToPsps/jython/stackbatch.py
===================================================================
--- branches/eam_branches/ipp-20130419/ippToPsps/jython/stackbatch.py (revision 35543)
+++ branches/eam_branches/ipp-20130419/ippToPsps/jython/stackbatch.py (revision 35554)
@@ -482,6 +482,6 @@
,X_PSF_SIG \
,Y_PSF_SIG \
- ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + " \
- ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + ")) / 1.085736) \
+ ,PSF_INST_FLUX / " + str(self.expTime) + " \
+ ,PSF_INST_FLUX_SIG / " + str(self.expTime) + " \
,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + str(self.expTime) + " \
,SKY / " + str(self.expTime) + " \
@@ -504,5 +504,5 @@
,MOMENTS_R1 \
,MOMENTS_RH \
- ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + " \
+ ,AP_FLUX / " + str(self.expTime) + " \
,NULL \
,KRON_FLUX / " + str(self.expTime) + " \
@@ -515,4 +515,9 @@
FROM SkyChip_psf"
+# these were used above (left over from before PSF_INST_FLUX, PSF_INST_FLUX_SIG were available?
+# ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + "
+# ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + ")) / 1.085736)
+# ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + "
+
# print "sql: ", sql
# response = raw_input("ready to insert stack det ")
@@ -526,4 +531,8 @@
self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0")
+ # XXX EAM : this seems quite inefficient : these commands use updates to set
+ # fields which are constant across all rows. Why not just include these in
+ # the original insert above?
+
self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID))
@@ -560,5 +569,6 @@
# response = raw_input("add primary key? ")
- if self.stackType == "DEEP_STACK":
+ ### XXX this should not be a DEEP_STACK only case...
+ if False and self.stackType == "DEEP_STACK":
#if deep stack and instFlux = null and err not null
@@ -572,4 +582,17 @@
# response = raw_input("add psf flux ")
+ # NOTE : Flux limits : in the current PSPS schema, negative fluxes
+ # cause problems for sql queries which work in mags as SQL cannot do
+ # something like (f < 0.0) ? -999 : -2.5*log10(f)
+ # as a result, the negative fluxes here result in floating point errors
+ sql = "UPDATE StackDetection SET psfFlux = 1e20 WHERE psfFlux <= 0.0"
+ self.scratchDb.execute(sql)
+
+ sql = "UPDATE StackDetection SET apFlux = 1e20 WHERE apFlux <= 0.0"
+ self.scratchDb.execute(sql)
+
+ sql = "UPDATE StackDetection SET kronFlux = 1e20 WHERE kronFlux <= 0.0"
+ self.scratchDb.execute(sql)
+
#leave null instflux in
#self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux")
@@ -930,4 +953,6 @@
def populatePspsTables(self):
+ self.skipBatch = False
+
if not self.useFullTables:
if not self.getIDsFromDVO():
Index: branches/eam_branches/ipp-20130419/ippToPsps/test/fulltest.sh
===================================================================
--- branches/eam_branches/ipp-20130419/ippToPsps/test/fulltest.sh (revision 35543)
+++ branches/eam_branches/ipp-20130419/ippToPsps/test/fulltest.sh (revision 35554)
@@ -1,3 +1,5 @@
#!/bin/csh -f
+
+chmod +x *.dvo *.sh
# a bit crude : get the datadir path by interpollation of the PATH element for psconfig
@@ -23,8 +25,28 @@
set objectbatch = 1
-set args = ""
-
-while ("$1" != "")
- switch ("$1")
+set targ = ($argv)
+set args = ""
+
+while ($#targ)
+ switch ("$targ[1]")
+ case -h
+ case -help
+ echo "USAGE: fulltest.sh [-XX] [+XX]"
+ echo " -XX : turn off features"
+ echo " +XX : turn on features"
+ echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, object"
+ exit 2
+ case -all
+ set mkgpc1 = 0
+ set initdb = 0
+ set initscratch = 0
+ set initbatch = 0
+ set camqueue = 0
+ set cambatch = 0
+ set stackqueue = 0
+ set stackbatch = 0
+ set objectqueue = 0
+ set objectbatch = 0
+ breaksw;
case -mkgpc1
set mkgpc1 = 0
@@ -52,9 +74,65 @@
breaksw;
default:
- set args=($args $1);
+ set args=($args $targ[1]);
breaksw;
endsw
- shift
+ shift targ
end
+
+set targ = ($args)
+set args = ""
+
+while ($#targ)
+ switch ("$targ[1]")
+ case +all
+ set mkgpc1 = 1
+ set initdb = 1
+ set initscratch = 1
+ set initbatch = 1
+ set camqueue = 1
+ set cambatch = 1
+ set stackqueue = 1
+ set stackbatch = 1
+ set objectqueue = 1
+ set objectbatch = 1
+ breaksw;
+ case +mkgpc1
+ set mkgpc1 = 1
+ breaksw;
+ case +initdb
+ set initdb = 1
+ breaksw;
+ case +initscratch
+ set initscratch = 1
+ breaksw;
+ case +initbatch
+ set initbatch = 1
+ breaksw;
+ case "+camera"
+ set camqueue = 1
+ set cambatch = 1
+ breaksw;
+ case +stack
+ set stackqueue = 1
+ set stackbatch = 1
+ breaksw;
+ case +object
+ set objectqueue = 1
+ set objectbatch = 1
+ breaksw;
+ default:
+ set args=($args $targ[1]);
+ breaksw;
+ endsw
+ shift targ
+end
+
+if ($args[1] != "") then
+ echo "USAGE: fulltest.sh [-XX] [+XX]"
+ echo " -XX : turn off features"
+ echo " +XX : turn on features"
+ echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, object"
+ exit 2
+endif
set stackqueuename = 0
Index: branches/eam_branches/ipp-20130419/ippToPsps/test/mkgpc1data.dvo
===================================================================
--- branches/eam_branches/ipp-20130419/ippToPsps/test/mkgpc1data.dvo (revision 35543)
+++ branches/eam_branches/ipp-20130419/ippToPsps/test/mkgpc1data.dvo (revision 35554)
@@ -37,6 +37,8 @@
macro mkfull
init.db
- mkcatdir.cam PS1_V3 PS1_V4
- mkcatdir.stk PS1_V3 PS1_V4
+ # mkcatdir.cam PS1_V3 PS1_V4
+ # mkcatdir.stk PS1_V3 PS1_V4
+ mkcatdir.cam PS1_V4 PS1_V4
+ mkcatdir.stk PS1_V4 PS1_V4
insert.stack.set
end