Index: /tags/ipp-20121026/Ohana/src/addstar/src/mkcmf.c =================================================================== --- /tags/ipp-20121026/Ohana/src/addstar/src/mkcmf.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/addstar/src/mkcmf.c (revision 34622) @@ -178,4 +178,6 @@ } + gauss_init (2048); + // load test stars from a file: Nstars = 0; @@ -254,6 +256,9 @@ gfits_modify (&header, "NASTRO", "%d", 1, 10); - gfits_modify (&header, "IMAGEID", "%d", 1, (int)(1000*drand48())); - gfits_modify (&header, "SOURCEID", "%d", 1, (int)(100*drand48())); + int imageID = 1000.0*drand48(); + int sourceID = 100.0*drand48(); + + gfits_modify (&header, "IMAGEID", "%d", 1, imageID); + gfits_modify (&header, "SOURCEID", "%d", 1, sourceID); PutCoords (&coords, &header); @@ -396,5 +401,4 @@ // this needs to make different output 'stars' entries depending on the desired type ALLOCATE (stars, PS1_DEV_0, Nstars); - gauss_init (2048); for (i = 0; i < Nstars; i++) { stars[i].detID = i; @@ -441,5 +445,4 @@ // this needs to make different output 'stars' entries depending on the desired type ALLOCATE (stars, PS1_DEV_1, Nstars); - gauss_init (2048); for (i = 0; i < Nstars; i++) { stars[i].detID = i; @@ -489,5 +492,4 @@ // this needs to make different output 'stars' entries depending on the desired type ALLOCATE (stars, CMF_PS1_V1, Nstars); - gauss_init (2048); for (i = 0; i < Nstars; i++) { stars[i].detID = i; @@ -537,5 +539,4 @@ // this needs to make different output 'stars' entries depending on the desired type ALLOCATE (stars, CMF_PS1_V2, Nstars); - gauss_init (2048); for (i = 0; i < Nstars; i++) { stars[i].detID = i; @@ -591,5 +592,4 @@ // this needs to make different output 'stars' entries depending on the desired type ALLOCATE (stars, CMF_PS1_V3, Nstars); - gauss_init (2048); for (i = 0; i < Nstars; i++) { stars[i].detID = i; @@ -649,5 +649,4 @@ // this needs to make different output 'stars' entries depending on the desired type ALLOCATE (stars, CMF_PS1_V4, Nstars); - gauss_init (2048); for (i = 0; i < Nstars; i++) { stars[i].detID = i; @@ -660,4 +659,11 @@ stars[i].M = M[i]; stars[i].Map = M[i] - 0.05; + stars[i].MapRaw = M[i] - 0.10; + + stars[i].Flux = flux; + stars[i].dFlux = flux * fSN; + + stars[i].apFlux = pow(10.0, -0.4*stars[i].Map); + stars[i].apFluxErr = stars[i].apFlux * fSN; if (ADDNOISE) { Index: /tags/ipp-20121026/Ohana/src/addstar/test/relphot.stacks.dvo =================================================================== --- /tags/ipp-20121026/Ohana/src/addstar/test/relphot.stacks.dvo (revision 34622) +++ /tags/ipp-20121026/Ohana/src/addstar/test/relphot.stacks.dvo (revision 34622) @@ -0,0 +1,323 @@ +# -*-sh-*- + +# relphot can set some secfilt values base on stack properties +# we would like to test that these are correctly set + +input tap.dvo + +# set globals +if (not($?PLOT)) set PLOT = 0 + +# set various global variables +macro init + $RA_CENTER = 10.0 + $DEC_CENTER = 20.0 + $PLATE_SCALE = 0.25 + $NSEASON = 3 + $NFILTER = 3 + $NCHIP_X = 2 + $NCHIP_Y = 2 + $NCELL_X = 2 + $NCELL_Y = 2 + $CHIP_DX = 1000 + $CHIP_DY = 1000 + + # images are loaded into dvo with GPC1 photcodes, so we need to get the nominal zps for those filters + $zpt_nominal:g = 24.58 + $zpt_nominal:r = 24.80 + $zpt_nominal:i = 24.74 + $zpt_nominal:z = 24.26 + $zpt_nominal:y = 23.41 + + # klam is negative, so klam*(airmass - 1) increase the zero point + $klam_nominal:g = -0.15 + $klam_nominal:r = -0.10 + $klam_nominal:i = -0.04 + $klam_nominal:z = -0.03 + $klam_nominal:y = -0.03 +end + +macro go + if ($0 != 3) + echo "go (fileroot) (catdir)" + break + end + + local i fileroot rootdir catdir + + $fileroot = $1 + $catdir = $2 + + dirname $1 -var rootdir + mkdir $rootdir + + exec rm -rf $catdir + + init + mkstars 500 + + # tapPLAN {((mjd_uc[]*4 + mjd_nc[]*4) + (mjd_uc[] + mjd_nc[]) + (mjd_uc[] + mjd_nc[])) / 3} + + # generate the basic images and check they were correctly ingested by dvo + mksequence $fileroot $catdir + break + + foreach filter g r i z y + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i raw + end + + # run relphot on the db and check that the images now match the expected values + exec relphot g,r,i,z,y -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir -nloop 0 -update >& tmp.log + foreach filter g r i z y + ckexposure catdir.test mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot + end + + tapDONE +end + +# go testdata/t1 catdir.test +# ckexposure catdir.test mjd_uc[0] zpt_uc[0] raw : result is mean of 0.0 (since test corrects for ZPT_NOMINAL vs REAL) +# ckexposure catdir.test mjd_nc[0] zpt_nc[0] raw : result is mean of 0.0 +# setphot -update -ubercal testzpt.fits -D CATDIR catdir.test +# ckexposure catdir.test mjd_uc[0] zpt_uc[0] corr : result is mean of 0.0 (since image is corrected to ZPT_REAL) +# ckexposure catdir.test mjd_nc[0] zpt_nc[0] corr : result is mean of -0.42 (since image is NOT corrected to ZPT_REAL) +# relphot g -region 9.5 10.5 19.5 20.5 -update -D CATDIR catdir.test -D STAR_TOOFEW 1 -D SIGMA_LIM 0.075 -statmode MEAN +# dvo: ckexposure catdir.test mjd_uc[0] zpt_uc[0] corr +# mean is now -0.42 +# dvo: ckexposure catdir.test mjd_nc[0] zpt_nc[0] corr +# mean is now -0.42 +# ** setphot assigned the zero point of the uc exposure to an effective -0.4 (ie, negative clouds), relphot was not told to keep this so it dragged the negative clouds to 0.0 -- this seems to be the right thing to do + +# re-run setphot, then relphot with -keep-ubercal + +# the test case has only 2 exposures, so the STAR_TOOFEW of 1 is necessary. Also, the mag and sigma distributions are somewhat artificial +# relphot g -v -region 9.5 10.5 19.5 20.5 -D CATDIR catdir.test -D STAR_TOOFEW 1 -D SIGMA_LIM 0.07 -statmode WT_MEAN -cloud-limit 0.5 -keep-ubercal -D IMAGE_OFFSET 0.5 -update +# dvo: ckexposure catdir.test mjd_uc[0] zpt_uc[0] corr +# mean is now 0.0 +# dvo: ckexposure catdir.test mjd_nc[0] zpt_nc[0] corr +# mean is now 0.0 +# ** setphot correctly assigned the zero point of UC & relphot adjusted NC to match + +macro ckexposure + if ($0 != 8) + echo "ckexposure (catdir) (mjd) (zpt) (exptime) (airmass) (filter) (mode)" + echo " mode == raw or corr" + break + end + + local CATDIR MJD_IMAGE ZPT_REAL ZPT_NOMINAL EXPTIME FILTER AIRMASS + + $CATDIR = $1 + $MJD_IMAGE = $2 + $ZPT_REAL = $3 + $EXPTIME = $4 + $AIRMASS = $5 + $FILTER = $6 + $MODE = $7 + + # XXX need a function to extract the nominal zpt for a given filter / photcode from the db + $ZPT_NOMINAL = $zpt_nominal:$FILTER + $KLAM_NOMINAL = $klam_nominal:$FILTER + + $TIMEFORMAT = mjd + $TIMEREF = 0.0 + + catdir $CATDIR + + # assume we still have stars_ra, stars_dec, stars_mag in hand + if ($PLOT) + dev -n 0 + region $RA_CENTER $DEC_CENTER 0.2 + images + pmeasure -all -m 15 20 + end + + skyregion {$RA_CENTER - 0.2/dcos($DEC_CENTER)} {$RA_CENTER + 0.2/dcos($DEC_CENTER)} {$DEC_CENTER - 0.2} {$DEC_CENTER + 0.2} + mextract ra dec mag xccd yccd where (abs(time - $MJD_IMAGE) < 0.0001) + + match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2 + + reindex stars_ra_m = stars_ra using index1 + reindex stars_dec_m = stars_dec using index1 + reindex stars_mag_m = stars_mag using index1 + set dr = 3600*(ra - stars_ra_m) + set dd = 3600*(dec - stars_dec_m) + set dm = mag - stars_mag_m + + if ($PLOT) + dev -n 1 + lim mag dm; clear; box; plot mag dm + end + + set cell_xbin = int(xccd / ($CHIP_DX / $NCELL_X)) + set cell_ybin = int(yccd / ($CHIP_DY / $NCELL_Y)) + subset dm00 = dm if (cell_xbin == 0) && (cell_ybin == 0) + subset dm01 = dm if (cell_xbin == 0) && (cell_ybin == 1) + subset dm10 = dm if (cell_xbin == 1) && (cell_ybin == 0) + subset dm11 = dm if (cell_xbin == 1) && (cell_ybin == 1) + + # uncorrected values behave like this: + # mag_DVO = m_inst + zpt_nominal + 2.5*log(exptime) + K*(airmass - 1.0) + # mag_real = m_inst + zpt_real + cell_offset + # dm = mag_DVO - mag_real = zpt_nominal - zpt_real - cell_offset + # - zpt_nominal + zpt_real + cell_offset ~ 0.0 + # zpt_real (in this case) = 25.0 + 2.5*log(exptime) + K*(airmass - 1.0) + # (actually, it is the value in the vector 'zpt' for this entry + # zpt_nominal = 24.58 + + # setphot-corrected values behave like this: + # mag_DVO = mag_real (because m_inst has cell_offset applied) + # dm = mag_DVO - mag_real ~ 0.0 + + $ZPT_REAL_NORM = $ZPT_REAL - 2.5*log($EXPTIME) - $KLAM_NOMINAL*($AIRMASS - 1.0) + + if ("$MODE" == "raw") + for ix 0 $NCELL_X + for iy 0 $NCELL_Y + # vstat -q dm$ix\$iy + # echo cell_off[$ix][$iy] {$MEDIAN - $ZPT_NOMINAL + $ZPT_REAL + cell_off[$ix][$iy]} $MEDIAN $MEAN $SIGMA + + set dm_adjust = dm$ix\$iy - $ZPT_NOMINAL + $ZPT_REAL_NORM + cell_off[$ix][$iy] + vstat -q dm_adjust + tapOK {abs($MEAN) < 0.005} "addstar raw ZP $MJD_IMAGE $FILTER" + end + end + return + end + + if ("$MODE" == "setphot_uc") + vstat -q dm + tapOK {abs($MEAN) < 0.005} "setphot_uc ZP $MJD_IMAGE $FILTER" + return + end + + if ("$MODE" == "setphot_nc") + vstat -q dm + tapOK {abs($MEAN - $ZPT_NOMINAL + $ZPT_REAL_NORM) < 0.005} "setphot_nc ZP $MJD_IMAGE $FILTER" + return + end + + if ("$MODE" == "relphot") + vstat -q dm + tapOK {abs($MEAN) < 0.005} "relphot ZP $MJD_IMAGE $FILTER" + return + end + +end + +macro mksequence + if ($0 != 3) + echo "mksequence (fileroot) (catdir)" + break + end + + local i + + # basic set of 5 stacks + foreach filter g r i z y + mkstack $1.$filter $RA_CENTER $DEC_CENTER $zpt_nominal:$filter 100.0 1.0 56228.666146 $filter $2 + end +end + +macro mkstack + if ($0 != 10) + echo "mkstack (fileroot) (ra) (dec) (zpt) (exptime) (airmass) (mjd) (filter) (catdir)" + break + end + + local ix iy date time datetime ra dec ROOT RAo DECo ZPT MJD FILTER EXPTIME AIRMASS CATDIR + + $ROOT = $1 + $RAo = $2 + $DECo = $3 + $ZPT = $4 + $EXPTIME = $5 + $AIRMASS = $6 + $MJD = $7 + $FILTER = $8 + $CATDIR = $9 + + $TIMEFORMAT = mjd + $TIMEREF = 0.0 + + for ix 0 $NCHIP_X + for iy 0 $NCHIP_Y + $dx = $CHIP_DX * $PLATE_SCALE * ($ix - 0.5*$NCHIP_X + 0.5) + $dy = $CHIP_DY * $PLATE_SCALE * ($iy - 0.5*$NCHIP_Y + 0.5) + $ra = $RAo - $dx / 3600.0 / dcos($DECo) + $dec = $DECo - $dy / 3600.0 + # echo $ra $dec $dx $dy + mkinput test.in.txt $ra $dec $ZPT + + # ra,dec is the center of this chip + local options + $options = + $options = $options -radec $ra $dec + $options = $options -type PS1_V4 + $options = $options -coords + $options = $options -photcode GPC1.$FILTER.SkyChip + $options = $options -no-noise + $options = $options -size $CHIP_DX $CHIP_DY + $options = $options -crpix {0.5*$CHIP_DX} {0.5*$CHIP_DY} + $options = $options -exptime $EXPTIME + + # these should not be needed for stacks, right + $options = $options -airmass $AIRMASS + $options = $options -mjd $MJD + + exec mkcmf test.in.txt $ROOT.$ix.$iy.cmf $options + echo mkcmf test.in.txt $ROOT.$ix.$iy.cmf $options + + # the fake images have inconsistent ra,dec and airmass,sidtime values + echo addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CATFORMAT PS1_V4 -D CAMERA gpc1 -quick-airmass + exec addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CATFORMAT PS1_V4 -D CAMERA gpc1 -quick-airmass >& tmp.log + end + end +end + +# make a simple input file for mkcmf +macro mkinput + if ($0 != 5) + echo "mkinput (filename) (ra) (dec) (zpt)" + break + end + + # chip coordinate of the stars + # ra,dec is at the chip center, which corresponds to pixel (CHIP_DX,CHIP_DY)/2 + set stars_X = (stars_ra - $2) * 3600.0 * dcos($3) / $PLATE_SCALE + $CHIP_DX*0.5 + set stars_Y = (stars_dec - $3) * 3600.0 / $PLATE_SCALE + $CHIP_DY*0.5 + set stars_M = (stars_mag - $4) + + set valid = (stars_X > 0) && (stars_X < $CHIP_DX) && (stars_Y > 0) && (stars_Y < $CHIP_DX) + subset stars_x = stars_X if valid + subset stars_y = stars_Y if valid + subset stars_mi = stars_M if valid + subset stars_r = stars_ra if valid + subset stars_d = stars_dec if valid + subset stars_mr = stars_mag if valid + + write -f "%10.6f %10.6f %7.3f %6.1f %6.1f %7.3f" $1 stars_r stars_d stars_mr stars_x stars_y stars_mi +end + +# generate a set of stars with raw RA, DEC, MAG values to use for all of the output cmf files +macro mkstars + if ($0 != 2) + echo "USAGE: mkstars (Nstars)" + break + end + + local RA_RANGE DEC_RANGE + + # the images are oriented along N-S, E-W lines + + # size of region of interest in linear arcseconds + $RA_RANGE = 1.25 * $CHIP_DX * $NCHIP_X * $PLATE_SCALE + $DEC_RANGE = 1.25 * $CHIP_DY * $NCHIP_Y * $PLATE_SCALE + + create tmp 0 $1 + set stars_ra = $RA_CENTER + $RA_RANGE * (rnd(tmp) - 0.5) / 3600 / dcos ($DEC_CENTER) + set stars_dec = $DEC_CENTER + $DEC_RANGE * (rnd(tmp) - 0.5) / 3600 + set stars_mag = 15.0 + 4.0 * rnd(tmp) + + # stars_ra,dec,mag are the true positions and mags of the stars +end Index: /tags/ipp-20121026/Ohana/src/libautocode/def/secfilt-ps1-v4.d =================================================================== --- /tags/ipp-20121026/Ohana/src/libautocode/def/secfilt-ps1-v4.d (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libautocode/def/secfilt-ps1-v4.d (revision 34622) @@ -6,21 +6,20 @@ # elements of data structure / FITS table -FIELD M, MAG, float, average mag in this band, mags -FIELD Map, MAG_AP, float, ave aperture mag in this band, mags -FIELD Mkron, MAG_KRON, float, ave kron mag in this band, mags -FIELD dMkron, MAG_KRON_ERR, float, formal error on average kron mag, mags -FIELD dM, MAG_ERR, float, formal error on average mag, mags -FIELD Xm, MAG_CHI, float, chisq on average mag, [100*log(value)] -FIELD FluxPSF, FLUX_PSF, float, mean flux psf fit (PS1: stack) -FIELD dFluxPSF, FLUX_PSF_ERR, float, mean flux psf error -FIELD FluxKron, FLUX_KRON, float, mean flux kron ap (PS1: stack) -FIELD dFluxKron, FLUX_KRON_ERR, float, mean flux kron err -FIELD flags, FLAGS, uint32_t, photometry flags -FIELD Ncode, NCODE, short, number of detections in band -FIELD Nused, NUSED, short, number of detections used in average -FIELD M_20, MAG_20, short, lower 20percent mag, millimags -FIELD M_80, MAG_80, short, upper 20percent mag, millimags -FIELD ubercalDist, UBERCAL_DIST, short, number of images from an ubercal-image -FIELD Mstdev, MAG_STDEV, short, standard deviation of measurements, millimags -FIELD stackID, STACK_ID, uint32_t, image ID of stack used for fluxes (if any) -FIELD dummy, DUMMY, uint32_t, padding +FIELD M, MAG, float, average mag in this band, mags +FIELD Map, MAG_AP, float, ave aperture mag in this band, mags +FIELD Mkron, MAG_KRON, float, ave kron mag in this band, mags +FIELD dMkron, MAG_KRON_ERR, float, formal error on average kron mag, mags +FIELD dM, MAG_ERR, float, formal error on average mag, mags +FIELD Xm, MAG_CHI, float, chisq on average mag, [100*log(value)] +FIELD FluxPSF, FLUX_PSF, float, mean flux psf fit (PS1: stack) +FIELD dFluxPSF, FLUX_PSF_ERR, float, mean flux psf error +FIELD FluxKron, FLUX_KRON, float, mean flux kron ap (PS1: stack) +FIELD dFluxKron, FLUX_KRON_ERR, float, mean flux kron err +FIELD flags, FLAGS, uint32_t, photometry flags +FIELD Ncode, NCODE, short, number of detections in band +FIELD Nused, NUSED, short, number of detections used in average +FIELD M_20, MAG_20, short, lower 20percent mag, millimags +FIELD M_80, MAG_80, short, upper 20percent mag, millimags +FIELD ubercalDist, UBERCAL_DIST, short, number of images from an ubercal-image +FIELD Mstdev, MAG_STDEV, short, standard deviation of measurements, millimags +FIELD stackDetectID, STACK_DETECT_ID, uint64_t, detection ID on stack used for fluxes (if any) Index: /tags/ipp-20121026/Ohana/src/libautocode/def/secfilt.d =================================================================== --- /tags/ipp-20121026/Ohana/src/libautocode/def/secfilt.d (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libautocode/def/secfilt.d (revision 34622) @@ -6,23 +6,22 @@ # elements of data structure / FITS table -FIELD M, MAG, float, average mag in this band, mags -FIELD Map, MAG, float, average mag in this band, mags -FIELD Mkron, MAG_KRON, float, ave kron mag in this band, mags -FIELD dMkron, MAG_KRON_ERR, float, formal error on average kron mag, mags -FIELD dM, MAG_ERR, float, formal error on average mag, mags -FIELD Xm, MAG_CHI, float, chisq on average mag, [100*log(value)] -FIELD FluxPSF, FLUX_PSF, float, mean flux psf fit (PS1: stack) -FIELD dFluxPSF, FLUX_PSF_ERR, float, mean flux psf error -FIELD FluxKron, FLUX_KRON, float, mean flux kron ap (PS1: stack) -FIELD dFluxKron, FLUX_KRON_ERR, float, mean flux kron err -FIELD flags, FLAGS, uint32_t, photometry flags -FIELD Ncode, NCODE, short, number of detections in band -FIELD Nused, NUSED, short, number of detections used in average -FIELD M_20, MAG_20, short, lower 20percent mag, millimags -FIELD M_80, MAG_80, short, upper 20percent mag, millimags -FIELD ubercalDist, UBERCAL_DIST, short, number of images from an ubercal-image -FIELD Mstdev, MAG_STDEV, short, standard deviation of measurements, millimags -FIELD stackID, STACK_ID, uint32_t, image ID of stack used for fluxes (if any) -FIELD dummy, DUMMY, uint32_t, padding +FIELD M, MAG, float, average mag in this band, mags +FIELD Map, MAG, float, average mag in this band, mags +FIELD Mkron, MAG_KRON, float, ave kron mag in this band, mags +FIELD dMkron, MAG_KRON_ERR, float, formal error on average kron mag, mags +FIELD dM, MAG_ERR, float, formal error on average mag, mags +FIELD Xm, MAG_CHI, float, chisq on average mag, [100*log(value)] +FIELD FluxPSF, FLUX_PSF, float, mean flux psf fit (PS1: stack) +FIELD dFluxPSF, FLUX_PSF_ERR, float, mean flux psf error +FIELD FluxKron, FLUX_KRON, float, mean flux kron ap (PS1: stack) +FIELD dFluxKron, FLUX_KRON_ERR, float, mean flux kron err +FIELD flags, FLAGS, uint32_t, photometry flags +FIELD Ncode, NCODE, short, number of detections in band +FIELD Nused, NUSED, short, number of detections used in average +FIELD M_20, MAG_20, short, lower 20percent mag, millimags +FIELD M_80, MAG_80, short, upper 20percent mag, millimags +FIELD ubercalDist, UBERCAL_DIST, short, number of images from an ubercal-image +FIELD Mstdev, MAG_STDEV, short, standard deviation of measurements, millimags +FIELD stackDetectID, STACK_DETECT_ID, uint64_t, detection ID on stack used for fluxes (if any) # *** 20090206 : new fields : M_20, M_80; dropped dummy Index: /tags/ipp-20121026/Ohana/src/libdvo/include/dvodb.h =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/include/dvodb.h (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/include/dvodb.h (revision 34622) @@ -45,5 +45,5 @@ MAG_80, MAG_UC_DIST, - MAG_STACK_ID, + MAG_STACK_DET_ID, MAG_FLUX_PSF, MAG_FLUX_PSF_ERR, Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dbExtractAverages.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dbExtractAverages.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dbExtractAverages.c (revision 34622) @@ -242,5 +242,5 @@ value.Flt = PhotUCdist (field->photcode, average, secfilt); break; - case MAG_STACK_ID: + case MAG_STACK_DET_ID: value.Int = PhotStackID (field->photcode, average, secfilt); break; Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dbFields.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dbFields.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dbFields.c (revision 34622) @@ -56,5 +56,5 @@ if (!strcasecmp (string, "80")) return (MAG_80); if (!strcasecmp (string, "ucdist")) return (MAG_UC_DIST); - if (!strcasecmp (string, "stackID")) return (MAG_STACK_ID); + if (!strcasecmp (string, "stackDetectID")) return (MAG_STACK_DET_ID); if (!strcasecmp (string, "fluxpsf")) return (MAG_FLUX_PSF); if (!strcasecmp (string, "fluxpsferr")) return (MAG_FLUX_PSF_ERR); @@ -260,5 +260,5 @@ field->magMode = mode; switch (mode) { - case MAG_STACK_ID: + case MAG_STACK_DET_ID: case MAG_NCODE: case MAG_NPHOT: @@ -351,5 +351,5 @@ field->magMode = mode; switch (mode) { - case MAG_STACK_ID: + case MAG_STACK_DET_ID: case MAG_NCODE: case MAG_NPHOT: Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_catalog.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_catalog.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_catalog.c (revision 34622) @@ -134,7 +134,5 @@ secfilt->ubercalDist = 1000; secfilt->Mstdev = NAN_S_SHORT; - secfilt->stackID = 0; - - secfilt->dummy = 0; + secfilt->stackDetectID = 0; } Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c (revision 34622) @@ -235,5 +235,5 @@ out[i].FluxKron = NAN; out[i].dFluxKron = NAN; - out[i].stackID = 0; + out[i].stackDetectID = 0; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 34622) @@ -229,5 +229,5 @@ out[i].FluxKron = NAN; out[i].dFluxKron = NAN; - out[i].stackID = 0; + out[i].stackDetectID = 0; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c (revision 34622) @@ -190,5 +190,5 @@ out[i].FluxKron = NAN; out[i].dFluxKron = NAN; - out[i].stackID = 0; + out[i].stackDetectID = 0; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 34622) @@ -233,5 +233,5 @@ out[i].FluxKron = NAN; out[i].dFluxKron = NAN; - out[i].stackID = 0; + out[i].stackDetectID = 0; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 34622) @@ -236,5 +236,5 @@ out[i].FluxKron = NAN; out[i].dFluxKron = NAN; - out[i].stackID = 0; + out[i].stackDetectID = 0; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c (revision 34622) @@ -234,5 +234,5 @@ out[i].FluxKron = NAN; out[i].dFluxKron = NAN; - out[i].stackID = 0; + out[i].stackDetectID = 0; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c (revision 34622) @@ -221,22 +221,22 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].Map = in[i].Map; - out[i].Mkron = in[i].Mkron; - out[i].dMkron = in[i].dMkron; - out[i].dM = in[i].dM; - out[i].Xm = in[i].Xm; - out[i].FluxPSF = in[i].FluxPSF; - out[i].dFluxPSF = in[i].dFluxPSF; - out[i].FluxKron = in[i].FluxKron; - out[i].dFluxKron = in[i].dFluxKron; - out[i].flags = in[i].flags; - out[i].Ncode = in[i].Ncode; - out[i].Nused = in[i].Nused; - out[i].M_20 = in[i].M_20; - out[i].M_80 = in[i].M_80; - out[i].Mstdev = in[i].Mstdev; - out[i].ubercalDist = in[i].ubercalDist; - out[i].stackID = in[i].stackID; + out[i].M = in[i].M; + out[i].Map = in[i].Map; + out[i].Mkron = in[i].Mkron; + out[i].dMkron = in[i].dMkron; + out[i].dM = in[i].dM; + out[i].Xm = in[i].Xm; + out[i].FluxPSF = in[i].FluxPSF; + out[i].dFluxPSF = in[i].dFluxPSF; + out[i].FluxKron = in[i].FluxKron; + out[i].dFluxKron = in[i].dFluxKron; + out[i].flags = in[i].flags; + out[i].Ncode = in[i].Ncode; + out[i].Nused = in[i].Nused; + out[i].M_20 = in[i].M_20; + out[i].M_80 = in[i].M_80; + out[i].Mstdev = in[i].Mstdev; + out[i].ubercalDist = in[i].ubercalDist; + out[i].stackDetectID = in[i].stackDetectID; } return (out); @@ -251,22 +251,22 @@ for (i = 0; i < Nvalues; i++) { - out[i].M = in[i].M; - out[i].Map = in[i].Map; - out[i].Mkron = in[i].Mkron; - out[i].dMkron = in[i].dMkron; - out[i].dM = in[i].dM; - out[i].Xm = in[i].Xm; - out[i].FluxPSF = in[i].FluxPSF; - out[i].dFluxPSF = in[i].dFluxPSF; - out[i].FluxKron = in[i].FluxKron; - out[i].dFluxKron = in[i].dFluxKron; - out[i].flags = in[i].flags; - out[i].Ncode = in[i].Ncode; - out[i].Nused = in[i].Nused; - out[i].M_20 = in[i].M_20; - out[i].M_80 = in[i].M_80; - out[i].Mstdev = in[i].Mstdev; - out[i].ubercalDist = in[i].ubercalDist; - out[i].stackID = in[i].stackID; + out[i].M = in[i].M; + out[i].Map = in[i].Map; + out[i].Mkron = in[i].Mkron; + out[i].dMkron = in[i].dMkron; + out[i].dM = in[i].dM; + out[i].Xm = in[i].Xm; + out[i].FluxPSF = in[i].FluxPSF; + out[i].dFluxPSF = in[i].dFluxPSF; + out[i].FluxKron = in[i].FluxKron; + out[i].dFluxKron = in[i].dFluxKron; + out[i].flags = in[i].flags; + out[i].Ncode = in[i].Ncode; + out[i].Nused = in[i].Nused; + out[i].M_20 = in[i].M_20; + out[i].M_80 = in[i].M_80; + out[i].Mstdev = in[i].Mstdev; + out[i].ubercalDist = in[i].ubercalDist; + out[i].stackDetectID = in[i].stackDetectID; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_elixir.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_elixir.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_elixir.c (revision 34622) @@ -244,5 +244,5 @@ out[i].FluxKron = NAN; out[i].dFluxKron = NAN; - out[i].stackID = 0; + out[i].stackDetectID = 0; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_loneos.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_loneos.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_loneos.c (revision 34622) @@ -237,5 +237,5 @@ out[i].FluxKron = NAN; out[i].dFluxKron = NAN; - out[i].stackID = 0; + out[i].stackDetectID = 0; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c (revision 34622) @@ -244,5 +244,5 @@ out[i].FluxKron = NAN; out[i].dFluxKron = NAN; - out[i].stackID = 0; + out[i].stackDetectID = 0; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c (revision 34622) @@ -244,5 +244,5 @@ out[i].FluxKron = NAN; out[i].dFluxKron = NAN; - out[i].stackID = 0; + out[i].stackDetectID = 0; } return (out); Index: /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_photcode_ops.c =================================================================== --- /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_photcode_ops.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/libdvo/src/dvo_photcode_ops.c (revision 34622) @@ -664,5 +664,5 @@ Ns = photcodes[0].hashNsec[code[0].code]; - ID = (Ns == -1) ? 0 : secfilt[Ns].stackID; + ID = (Ns == -1) ? 0 : secfilt[Ns].stackDetectID; return (ID); } Index: /tags/ipp-20121026/Ohana/src/relphot/src/StarOps.c =================================================================== --- /tags/ipp-20121026/Ohana/src/relphot/src/StarOps.c (revision 34621) +++ /tags/ipp-20121026/Ohana/src/relphot/src/StarOps.c (revision 34622) @@ -401,9 +401,9 @@ float stackCenterOffsetMin = 1e9; - int stackCenterIDmin = -1; + // int stackCenterIDmin = -1; off_t stackCenterMeasureMin = -1; float stackPrimaryOffsetMin = 1e9; - int stackPrimaryIDmin = -1; + // int stackPrimaryIDmin = -1; off_t stackPrimaryMeasureMin = -1; @@ -497,5 +497,5 @@ if (stackPrimaryOffset < stackPrimaryOffsetMin) { stackPrimaryOffsetMin = stackPrimaryOffset; - stackPrimaryIDmin = stackImageID; + // stackPrimaryIDmin = stackImageID; stackPrimaryMeasureMin = m; } @@ -506,5 +506,5 @@ if (stackCenterOffset < stackCenterOffsetMin) { stackCenterOffsetMin = stackCenterOffset; - stackCenterIDmin = stackImageID; + // stackCenterIDmin = stackImageID; stackCenterMeasureMin = m; } @@ -650,5 +650,8 @@ if (haveStack) { m = (stackPrimaryMeasureMin >= 0) ? stackPrimaryMeasureMin : stackCenterMeasureMin; - ID = (stackPrimaryMeasureMin >= 0) ? stackPrimaryIDmin : stackCenterIDmin; + + // we are now populating stackDetectID not stack Image ID in secfilt + // ID = (stackPrimaryMeasureMin >= 0) ? stackPrimaryIDmin : stackCenterIDmin; + ID = catalog[Nc].measure[m].extID; // for the stack, this is the stackDetectID // get the zero point for the selected image @@ -693,5 +696,5 @@ catalog[Nc].secfilt[Nsecfilt*j+Nsec].dFluxKron = zpFactor * catalog[Nc].measure[m].dFluxKron; - catalog[Nc].secfilt[Nsecfilt*j+Nsec].stackID = ID; + catalog[Nc].secfilt[Nsecfilt*j+Nsec].stackDetectID = ID; catalog[Nc].measure[m].dbFlags |= ID_MEAS_STACK_PHOT_SRC; Index: /tags/ipp-20121026/ippToPsps/config/OB/tables.vot =================================================================== --- /tags/ipp-20121026/ippToPsps/config/OB/tables.vot (revision 34621) +++ /tags/ipp-20121026/ippToPsps/config/OB/tables.vot (revision 34622) @@ -45,13 +45,4 @@ Declination error - - Cartesian x - - - Cartesian y - - - Cartesian z - ecliptic longitude @@ -66,10 +57,4 @@ galactic latitude - - supergalactic longitude - - - supergalactic latitude - total number of detection measurements in all filters @@ -201,8 +186,8 @@ 80 pct w magnitude - + g stack psf flux - + estimated error in g stack psf flux @@ -216,8 +201,11 @@ flags for g stack - + + ODM detection identifier + + r stack psf flux - + estimated error in r stack psf flux @@ -231,8 +219,11 @@ flags for r stack - + + ODM detection identifier + + i stack psf flux - + estimated error in i stack psf flux @@ -246,8 +237,11 @@ flags for i stack - + + ODM detection identifier + + z stack psf flux - + estimated error in z stack psf flux @@ -261,8 +255,11 @@ flags for z stack - + + ODM detection identifier + + y band stack PSF flux - + y band stack psf flux err @@ -276,6 +273,9 @@ flags for y stack - - currently unused + + ODM detection identifier + + + qf perfect Index: /tags/ipp-20121026/ippToPsps/config/P2/tables.vot =================================================================== --- /tags/ipp-20121026/ippToPsps/config/P2/tables.vot (revision 34621) +++ /tags/ipp-20121026/ippToPsps/config/P2/tables.vot (revision 34622) @@ -404,8 +404,8 @@ estimated error in y - + PSF instrumental flux - + estimated error in flux @@ -425,6 +425,6 @@ PSF likelihood - - PSF coverage factor + + PSF qf @@ -437,14 +437,14 @@ momeny YY - - Aperture magnitude - - - Aperture magnitude error - - + + Aperture Flux + + + Aperture Flux error + + Kron flux - + Kron flux error @@ -455,17 +455,17 @@ Kron radius error - - Likelihood the source is a cosmic ray - - - Likelihood the source is extended + + psf QF Perfect + + + PSF Chi sq flag indicating provenance information - + PSF sky level at source - + estimated error in sky @@ -539,4 +539,56 @@ + + VOTable description of PSPS table DetectionCalib + + + ODM object identifier index + + + ODM detection identifier index + + + IPP object number + + + detection ID generated by IPP + + + filter identifier + + + survey identifier + + + right ascension + + + declination + + + right ascension error + + + declination error + + + zero point offset + + + zerop point offset error + + + exposure time + + + airmass + + + Data release to propagate to the object + + + + +
Index: /tags/ipp-20121026/ippToPsps/config/ST/tables.vot =================================================================== --- /tags/ipp-20121026/ippToPsps/config/ST/tables.vot (revision 34621) +++ /tags/ipp-20121026/ippToPsps/config/ST/tables.vot (revision 34622) @@ -148,7 +148,4 @@ projection cell identifier - - Time of mid observation (unit = day) - measured x on CCD from PSF fit @@ -163,8 +160,8 @@ estimated error in y - + PSF instrumental flux - + estimated error in instrumental flux @@ -193,7 +190,10 @@ PSF likelihood - - PSF coverage factor - + + PSF Q factor + + + PSF QF Perfect + moment XX (unit = arcsec) @@ -223,21 +223,21 @@ moment RH (unit = arcsec) - - Aperture magnitude (unit = magnitude) - - - Aperture magnitude error (unit = magnitude) + + Aperture flux (unit = adu/s) + + + Aperture flux error (unit = adu/s) - Kron flux (unit = magnitude) + Kron flux (unit = adu/s) - Kron flux error (unit = magnitude) + Kron flux error (unit = adu/s) Kron radius (unit = arcsecs) - - Kron radius error (unit = arcsecs) + + psf chi sq @@ -293,9 +293,9 @@ stack identifier - - isophotal magnitude - - - estimated error in isophotal magnitude + + isophotal flux + + + estimated error in isophotal flux @@ -332,9 +332,9 @@ estimated error inPetrosian radius - - Petrosian magntiude - - - estimated error in petMag + + Petrosian Flux + + + estimated error in petFlux @@ -765,8 +765,8 @@ estimated error in deVaucouleurs radius - + deVaucouleurs magntiude - + estimated error in deV_mag @@ -891,9 +891,9 @@ estimated error in Exponential fit radius - - Exponential fit magntiude - - - estimated error in expMag + + Exponential fit flux + + + estimated error in expFlux @@ -1017,9 +1017,9 @@ estimated error in Sersic radius - - Sersic magntiude - - - estimated error in serMag + + Sersic Flux + + + estimated error in serFlux @@ -1240,4 +1240,56 @@ + + VOTable description of PSPS table StackDetectionCalib + + + ODM object identifier index + + + ODM detection identifier index + + + IPP object number + + + detection ID generated by IPP + + + filter identifier + + + survey identifier + + + right ascension + + + declination + + + right ascension error + + + declination error + + + zero point offset + + + zerop point offset error + + + exposure time + + + airmass + + + Data release to propagate to the object + + + + +
Index: /tags/ipp-20121026/ippToPsps/jython/detectionbatch.py =================================================================== --- /tags/ipp-20121026/ippToPsps/jython/detectionbatch.py (revision 34621) +++ /tags/ipp-20121026/ippToPsps/jython/detectionbatch.py (revision 34622) @@ -388,8 +388,11 @@ # delete all detections with PSF_INST_MAG < 17.5, as decreed by Gene BEFORE = self.scratchDb.getRowCount(ippTableName) - sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5" - self.scratchDb.execute(sql) + #don't do this anymore + #sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5" + #self.scratchDb.execute(sql) #DELETED = BEFORE - self.scratchDb.getRowCount(ippTableName) - results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName) + #results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName) + #we aren't deletng these now + results['SATDET'] = 0 #self.logger.infoPair("Saturated detections", "%d deleted" % DELETED) @@ -404,6 +407,6 @@ ,xPosErr \ ,yPosErr \ - ,instFlux \ - ,instFluxErr \ + ,psfFlux \ + ,psfFluxErr \ ,peakADU \ ,psfWidMajor \ @@ -411,11 +414,13 @@ ,psfTheta \ ,psfLikelihood \ - ,psfCf \ + ,psfQf \ ,momentXX \ ,momentXY \ ,momentYY \ - ,apMag \ + ,apFlux \ ,kronFlux \ ,kronFluxErr \ + ,psfQfPerfect \ + ,psfChiSq \ ,infoFlag \ ,sky \ @@ -450,5 +455,7 @@ ,KRON_FLUX / " + self.header['EXPTIME'] + " \ ,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \ - ,FLAGS\ + ,PSF_QF_PERFECT \ + ,PSF_CHISQ \ + ,FLAGS2 << 32 | FLAGS\ ,SKY \ ,SKY_SIGMA \ @@ -459,19 +466,36 @@ , " + str(self.config.dataRelease) + "\ FROM " + ippTableName + # SA10 todo + # need to correct formulas for psfFlux/psfFluxErr + # need to correct apMag apFlux + # need to correct unites kronFlux/Err to correct units + # need to change psfqfperfect -> psfqfperfect PSF_QF_PERFECT? + # need to change psfchisq PSF_CHI_SQ? + # need to change unites of sky/skyErr to adu/s/pixel + # need formula for sgsep/psflikelhood + # self.logger.info(sql) self.scratchDb.execute(sql) # now delete bad flux and bad chip positions - sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux - self.scratchDb.execute(sql) - + #sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux + #self.scratchDb.execute(sql) + + # add a instFlux = 0.0 -> 0.000001 + sql="UPDATE " + pspsTableName + " SET psfFlux = 0.0000001 WHERE psfFlux = 0" + self.scratchDb.execute(sql) + # update cosmic ray and extended likelihoods - sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0" - self.scratchDb.execute(sql) - sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0" - self.scratchDb.execute(sql) + #sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0" + #self.scratchDb.execute(sql) + #sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0" + #self.scratchDb.execute(sql) # remove detections will NULL inst flux or NULL peak ADU - results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux") - results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU") + # leave them in at this stage- don't delete them. + #results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux") + #results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU") + #we don't delete thos anymore + results['NULLINSTFLUX'] = 0; + results['NULLPEAKADU'] = 0; ''' @@ -504,4 +528,41 @@ ''' + Populates the DetectionCalib table for this OTA + ''' + def populateDetectionCalibTable(self, ota): + + tableName = "DetectionCalib_" + ota + # drop then re-create table + self.scratchDb.dropTable(tableName) + sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib" + try: self.scratchDb.execute(sql) + except: pass + + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + objID \ + ,detectID \ + ,ippObjID \ + ,ippDetectID \ + ,filterID \ + ,surveyID \ + ) \ + SELECT \ + objID \ + ,detectID \ + ,ippObjID \ + ,ippDetectID \ + ,filterID \ + ,surveyID \ + FROM Detection_" + ota + self.scratchDb.execute(sql) + #SA10 TODO + # insert ra/dec/calstuff into Table + # heather discovered dec is called 'dec_' + sql = "UPDATE " + tableName + " SET ra = -999, dec_ =-999, raErr = -999, decErr = -999, zp = -999, zpErr = -999, expTime = -999, airMass = -999" + self.scratchDb.execute(sql) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) + + ''' Populates the Detection table for this OTA ''' @@ -608,5 +669,5 @@ extension = "XY%d%d_psf" % (x, y) self.scratchDb.createIndex(extension, "IPP_IDET") - + self.logger.infoPair("created indexes on", "IPP tables") ''' Updates provided table with DVO IDs from DVO table @@ -620,5 +681,5 @@ a.detectID = b.detectID, \ a.objID = b.objID, \ - a.infoFlag = b.flags << 32 | a.infoFlag \ + a.infoFlag = b.flags << 46 | a.infoFlag \ WHERE a.ippDetectID = b.ippDetectID \ AND b.imageID = " + str(imageID) @@ -658,13 +719,17 @@ self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % ota) continue - + # self.logger.info("populate stuff "); # populate remainder of tables self.populateDetectionTable(ota, results) - + # self.logger.info("successful populate "); # now add DVO IDs self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota]) + # self.logger.info("updated dvoids") results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") - self.updateImageID("Detection_" + ota, x, y) + #self.logger.info("deleted nulls") + self.updateImageID("Detection_" + ota, x, y) + # self.logger.info("updateImageId") rowCount = self.scratchDb.getRowCount("Detection_" + ota) + # self.logger.info("got row count") self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", ota, @@ -675,5 +740,4 @@ results['NULLOBJID'], rowCount) - totalOriginal = totalOriginal + results['ORIGINALTOTAL'] totalSatDet = totalSatDet + results['SATDET'] @@ -682,5 +746,5 @@ totalNullObjID = totalNullObjID + results['NULLOBJID'] totalDetections = totalDetections + rowCount - + #self.logger.info("updated totals") # check we have something in this Detection table TODO add this to table above if rowCount < 1: @@ -692,8 +756,13 @@ SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota])) self.scratchDb.execute(sql) - + + #self.logger.info("updated imagedata") self.populateSkinnyObjectTable(ota) + #self.logger.info("updated skinnyobject") self.populateObjectCalColorTable(ota) - + #self.logger.info("updated objectcalcolor") + self.populateDetectionCalibTable(ota) + #self.logger.info("updated detectioncalibtable") + # add these to list of tables to export later self.tablesToExport.append("ImageMeta_" + ota) @@ -701,8 +770,9 @@ self.tablesToExport.append("SkinnyObject_" + ota) self.tablesToExport.append("ObjectCalColor_" + ota) + self.tablesToExport.append("DetectionCalib_" + ota) tables.append("Detection_" + ota) otaCount = otaCount + 1 - + #self.logger.info("end of ota") # print totals self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") @@ -757,5 +827,5 @@ else : regex = ".*.psf" - columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS SKY SKY_SIGMA EXT_NSIGMA" + columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" return super(DetectionBatch, self).importIppTables(columns, regex) Index: /tags/ipp-20121026/ippToPsps/jython/objectbatch.py =================================================================== --- /tags/ipp-20121026/ippToPsps/jython/objectbatch.py (revision 34621) +++ /tags/ipp-20121026/ippToPsps/jython/objectbatch.py (revision 34622) @@ -114,6 +114,6 @@ for filter in filters: - sql = "UPDATE Object \ - JOIN " + cpsTable + " AS cps ON (cps.row = (Object.row*" + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + sql = "UPDATE Object JOIN \ + " + cpsTable + " AS cps ON (cps.row = (Object.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ SET \ n" + filter[1] + " = NCODE \ @@ -122,14 +122,28 @@ ," + filter[1] + "MeanKronMag = MAG_KRON \ ," + filter[1] + "MeanKronMagErr = MAG_KRON_ERR \ - ," + filter[1] + "StackPSFFlux = FLUX_PSF \ - ," + filter[1] + "StackPSFFluxErr = FLUX_PSF_ERR \ + ," + filter[1] + "StackPSFMag = -2.5 * log10(FLUX_PSF) + 8.9 \ + ," + filter[1] + "StackPSFMagErr = FLUX_PSF_ERR / FLUX_PSF \ ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) + 8.9 \ ," + filter[1] + "StackKronMagErr = FLUX_KRON_ERR / FLUX_KRON \ ," + filter[1] + "20pct = MAG_20/1000 \ ," + filter[1] + "80pct = MAG_80/1000 \ - ," + filter[1] + "Flags = FLAGS " - + ," + filter[1] + "Flags = FLAGS \ + ," + filter[1] + "StackDetectID = STACK_ID" + + #stackDetectID needs to be filled still for SA10 - currently at STACK_ID for test purposes + # everthing should be correctly calculated as mags for SA10 + #self.logger.info(sql) + self.scratchDb.execute(sql) + + self.logger.infoPair("Calculating nDetections from", "n[filters]") + for filter in filters: + # now do a sum of n[filters], but do not include the ones with -999 + sql = "UPDATE Object \ + SET nDetections = nDetections + n" + filter[1] + " \ + WHERE n" + filter[1] + " != -999" + self.scratchDb.execute(sql) - self.scratchDb.execute(sql) + + # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12) @@ -204,5 +218,5 @@ ,decErr \ ,nDetections \ - ,extra \ + ,qfPerfect \ ,sgSep \ ,dataRelease \ Index: /tags/ipp-20121026/ippToPsps/jython/stackbatch.py =================================================================== --- /tags/ipp-20121026/ippToPsps/jython/stackbatch.py (revision 34621) +++ /tags/ipp-20121026/ippToPsps/jython/stackbatch.py (revision 34622) @@ -89,4 +89,5 @@ self.scratchDb.dropTable("SkinnyObject") self.scratchDb.dropTable("ObjectCalColor") + self.scratchDb.dropTable("StackDetectionCalib") # delete IPP tables @@ -140,43 +141,43 @@ sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET \ - "+prefix+"flxR1=b.APER_FLUX_1, \ - "+prefix+"flxR1Err=b.APER_FLUX_ERR_1, \ - "+prefix+"flxR1Std=b.APER_FLUX_STDEV_1, \ + "+prefix+"flxR1=b.APER_FLUX_1 / " + str(self.expTime) + ", \ + "+prefix+"flxR1Err=b.APER_FLUX_ERR_1 / " + str(self.expTime) + ", \ + "+prefix+"flxR1Std=b.APER_FLUX_STDEV_1 / " + str(self.expTime) + ", \ "+prefix+"flxR1Fill=b.APER_FILL_1, \ - "+prefix+"flxR2=b.APER_FLUX_2, \ - "+prefix+"flxR2Err=b.APER_FLUX_ERR_2, \ - "+prefix+"flxR2Std=b.APER_FLUX_STDEV_2, \ + "+prefix+"flxR2=b.APER_FLUX_2 / " + str(self.expTime) + ", \ + "+prefix+"flxR2Err=b.APER_FLUX_ERR_2 / " + str(self.expTime) + ", \ + "+prefix+"flxR2Std=b.APER_FLUX_STDEV_2 / " + str(self.expTime) + ", \ "+prefix+"flxR2Fill=b.APER_FILL_2, \ - "+prefix+"flxR3=b.APER_FLUX_3, \ - "+prefix+"flxR3Err=b.APER_FLUX_ERR_3, \ - "+prefix+"flxR3Std=b.APER_FLUX_STDEV_3, \ + "+prefix+"flxR3=b.APER_FLUX_3 / " + str(self.expTime) + ", \ + "+prefix+"flxR3Err=b.APER_FLUX_ERR_3 / " + str(self.expTime) + ", \ + "+prefix+"flxR3Std=b.APER_FLUX_STDEV_3 / " + str(self.expTime) + ", \ "+prefix+"flxR3Fill=b.APER_FILL_3, \ - "+prefix+"flxR4=b.APER_FLUX_4, \ - "+prefix+"flxR4Err=b.APER_FLUX_ERR_4, \ - "+prefix+"flxR4Std=b.APER_FLUX_STDEV_4, \ + "+prefix+"flxR4=b.APER_FLUX_4 / " + str(self.expTime) + ", \ + "+prefix+"flxR4Err=b.APER_FLUX_ERR_4 / " + str(self.expTime) + ", \ + "+prefix+"flxR4Std=b.APER_FLUX_STDEV_4 / " + str(self.expTime) + ", \ "+prefix+"flxR4Fill=b.APER_FILL_4, \ - "+prefix+"flxR5=b.APER_FLUX_5, \ - "+prefix+"flxR5Err=b.APER_FLUX_ERR_5, \ - "+prefix+"flxR5Std=b.APER_FLUX_STDEV_5, \ + "+prefix+"flxR5=b.APER_FLUX_5 / " + str(self.expTime) + ", \ + "+prefix+"flxR5Err=b.APER_FLUX_ERR_5 / " + str(self.expTime) + ", \ + "+prefix+"flxR5Std=b.APER_FLUX_STDEV_5 / " + str(self.expTime) + ", \ "+prefix+"flxR5Fill=b.APER_FILL_5, \ - "+prefix+"flxR6=b.APER_FLUX_6, \ - "+prefix+"flxR6Err=b.APER_FLUX_ERR_6, \ - "+prefix+"flxR6Std=b.APER_FLUX_STDEV_6, \ + "+prefix+"flxR6=b.APER_FLUX_6 / " + str(self.expTime) + ", \ + "+prefix+"flxR6Err=b.APER_FLUX_ERR_6 / " + str(self.expTime) + ", \ + "+prefix+"flxR6Std=b.APER_FLUX_STDEV_6 / " + str(self.expTime) + ", \ "+prefix+"flxR6Fill=b.APER_FILL_6, \ - "+prefix+"flxR7=b.APER_FLUX_7, \ - "+prefix+"flxR7Err=b.APER_FLUX_ERR_7, \ - "+prefix+"flxR7Std=b.APER_FLUX_STDEV_7, \ + "+prefix+"flxR7=b.APER_FLUX_7 / " + str(self.expTime) + ", \ + "+prefix+"flxR7Err=b.APER_FLUX_ERR_7 / " + str(self.expTime) + ", \ + "+prefix+"flxR7Std=b.APER_FLUX_STDEV_7 / " + str(self.expTime) + ", \ "+prefix+"flxR7Fill=b.APER_FILL_7, \ - "+prefix+"flxR8=b.APER_FLUX_8, \ - "+prefix+"flxR8Err=b.APER_FLUX_ERR_8, \ - "+prefix+"flxR8Std=b.APER_FLUX_STDEV_8, \ + "+prefix+"flxR8=b.APER_FLUX_8 / " + str(self.expTime) + ", \ + "+prefix+"flxR8Err=b.APER_FLUX_ERR_8 / " + str(self.expTime) + ", \ + "+prefix+"flxR8Std=b.APER_FLUX_STDEV_8 / " + str(self.expTime) + ", \ "+prefix+"flxR8Fill=b.APER_FILL_8, \ - "+prefix+"flxR9=b.APER_FLUX_9, \ - "+prefix+"flxR9Err=b.APER_FLUX_ERR_9, \ - "+prefix+"flxR9Std=b.APER_FLUX_STDEV_9, \ + "+prefix+"flxR9=b.APER_FLUX_9 / " + str(self.expTime) + ", \ + "+prefix+"flxR9Err=b.APER_FLUX_ERR_9 / " + str(self.expTime) + ", \ + "+prefix+"flxR9Std=b.APER_FLUX_STDEV_9 / " + str(self.expTime) + ", \ "+prefix+"flxR9Fill=b.APER_FILL_9, \ - "+prefix+"flxR10=b.APER_FLUX_10, \ - "+prefix+"flxR10Err=b.APER_FLUX_ERR_10, \ - "+prefix+"flxR10Std=b.APER_FLUX_STDEV_10, \ + "+prefix+"flxR10=b.APER_FLUX_10 / " + str(self.expTime) + ", \ + "+prefix+"flxR10Err=b.APER_FLUX_ERR_10 / " + str(self.expTime) + ", \ + "+prefix+"flxR10Std=b.APER_FLUX_STDEV_10 / " + str(self.expTime) + ", \ "+prefix+"flxR10Fill=b.APER_FILL_10 \ WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition @@ -196,6 +197,6 @@ sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ "+model+"Radius=b.EXT_WIDTH_MAJ, \ - "+model+"Mag=b.EXT_INST_MAG, \ - "+model+"MagErr=b.EXT_INST_MAG_SIG, \ + "+model+"Flux=b.EXT_INST_MAG, \ + "+model+"FluxErr=b.EXT_INST_MAG_SIG, \ "+model+"Ab=b.EXT_WIDTH_MAJ/b.EXT_WIDTH_MIN, \ "+model+"Phi=b.EXT_THETA, \ @@ -229,4 +230,6 @@ "+modelLong+"Covar77=b.EXT_COVAR_06_06 \ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" + + #SA10 convert mag to flux in calculation above self.scratchDb.execute(sql) @@ -319,11 +322,10 @@ ,skyCellID \ ,projectionCellID \ - ,obsTime \ ,xPos \ ,yPos \ ,xPosErr \ ,yPosErr \ - ,instFlux \ - ,instFluxErr \ + ,psfFlux \ + ,psfFluxErr \ ,peakFlux \ ,sky \ @@ -335,5 +337,6 @@ ,psfLikelihood \ ,infoFlag \ - ,psfCf \ + ,psfQf \ + ,psfQfPerfect \ ,momentXX \ ,momentXY \ @@ -345,10 +348,10 @@ ,momentR1 \ ,momentRH \ - ,apMag \ - ,apMagErr \ + ,apFlux \ + ,apFluxErr \ ,kronFlux \ ,kronFluxErr \ ,kronRad \ - ,kronRadErr \ + ,psfChiSq \ ,nFrames \ ,assocDate \ @@ -359,5 +362,4 @@ ," + self.skycell + " \ ," + self.projectioncell + " \ - ," + self.header['MJD-OBS'] + " \ ,X_PSF \ ,Y_PSF \ @@ -374,6 +376,7 @@ ,PSF_THETA \ ,psfLikelihood(EXT_NSIGMA) \ - ,FLAGS2 << 32 | FLAGS \ + , FLAGS2 << 32 | FLAGS \ ,PSF_QF \ + ,PSF_QF_PERFECT \ ,MOMENTS_XX \ ,MOMENTS_XY \ @@ -389,10 +392,14 @@ ,KRON_FLUX / " + str(self.expTime) + " \ ,KRON_FLUX_ERR / " + str(self.expTime) + " \ - , MOMENTS_R1 * 2.5 \ - , NULL \ + ,MOMENTS_R1 * 2.5 \ + ,PSF_CHISQ \ ,N_FRAMES \ , '" + self.dateStr + "' \ ," + self.historyModNum + " \ FROM SkyChip_psf" + + #SA10 to do + # put in correct formula for apFlux, psfFlux, psfLikelihood, sgsep + # need correct units of anything in Flux self.scratchDb.execute(sql) @@ -417,5 +424,5 @@ self.updateStackTypeID("StackDetection") - self.updateDvoIDs("StackDetection") + self.updateDvoIDsAndFlags("StackDetection") sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)" @@ -427,6 +434,6 @@ #if deep stack and instFlux = null and err not null sql = "UPDATE StackDetection AS a, SkyChip_psf AS b \ - SET instFlux = 2*b.PSF_INST_FLUX_SIG / " + str(self.expTime) + " \ - WHERE instFlux IS NULL \ + SET psfFlux = 2*b.PSF_INST_FLUX_SIG / " + str(self.expTime) + " \ + WHERE psfFlux IS NULL \ AND a.ippDetectID = b.IPP_IDET \ AND b.PSF_INST_FLUX_SIG IS NOT NULL" @@ -434,6 +441,6 @@ self.scratchDb.execute(sql) - - self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux") + #leave null instflux in + #self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux") self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID") @@ -474,6 +481,6 @@ petRadius=b.PETRO_RADIUS \ ,petRadiusErr=b.PETRO_RADIUS_ERR \ - ,petMag=b.PETRO_MAG \ - ,petMagErr=b.PETRO_MAG_ERR \ + ,petFlux=b.PETRO_MAG \ + ,petFluxErr=b.PETRO_MAG_ERR \ ,petR50=b.PETRO_RADIUS_50 \ ,petR50Err=b.PETRO_RADIUS_50_ERR \ @@ -482,4 +489,5 @@ ,petCf=b.PETRO_FILL \ WHERE a.ippDetectID=b.IPP_IDET" + #SA10 Put correct calculation for PetFlux self.scratchDb.execute(sql) @@ -586,4 +594,40 @@ self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID)) self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.config.dataRelease)) + + ''' + Populates the StackDetectionCalib table + ''' + def populateStackDetectionCalib(self): + self.logger.infoPair("Processing table", "StackDetectionCalib") + tableName = "StackDetectionCalib" + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + objID \ + ,stackDetectID \ + ,ippObjID \ + ,ippDetectID \ + ,filterID \ + ,surveyID \ + ) \ + SELECT \ + objID \ + ,stackDetectID \ + ,ippObjID \ + ,ippDetectID \ + ,filterID \ + ,surveyID \ + FROM StackDetection" + self.logger.info(sql) + self.scratchDb.execute(sql) + #SA10 TODO + # insert ra/dec/calstuff into Table + # heather discovered dec is called 'dec_' + sql = "UPDATE " + tableName + " SET ra = -999, dec_ =-999, raErr = -999, decErr = -999, zp = -999, zpErr = -999, expTime = -999, airMass = -999" + self.logger.info(sql) + self.scratchDb.execute(sql) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) + + + ''' @@ -649,7 +693,24 @@ WHERE a.ippDetectID = b.ippDetectID \ AND b.imageID = " + str(imageID) - - self.scratchDb.execute(sql) - + self.scratchDb.execute(sql) + + ''' + Updates Flags + ''' + def updateDvoIDsAndFlags(self, table): + self.logger.infoPair("getting","imageID") + imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) + self.logger.infoPair("obtained","imageID") + self.logger.debug("Updating table '" + table + "' with DVO IDs...") + sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ + a.ippObjID = b.ippObjID, \ + a.stackDetectID = 10 * b.detectID + " + str(self.filterID) + ", \ + a.objID = b.objID, \ + a.infoFlag = b.flags << 46 | a.infoFlag \ + WHERE a.ippDetectID = b.ippDetectID \ + AND b.imageID = " + str(imageID) + self.scratchDb.execute(sql) + + ''' Does the processing, i.e. pulling stuff from IPP tables into PSPS tables @@ -660,15 +721,22 @@ if not self.getIDsFromDVO(): return False - + #self.logger.infoPair("populating","StackMeta") self.populateStackMeta() + #self.logger.infoPair("populating","StackDetection") self.populateStackDetection() - + if self.stackType != "NIGHTLY_STACK": + #self.logger.infoPair("populating","StackModelFit") self.populateStackModelFit() + #self.logger.infoPair("populating","StackApFlx") self.populateStackApFlx() - + #self.logger.infoPair("populating","StackToImage") self.populateStackToImage() + #self.logger.infoPair("populating","SkinnyObject") self.populateSkinnyObject() + #self.logger.infoPair("populating","ObjectCalColor") self.populateObjectCalColor() + #self.logger.infoPair("populating","StackDetectionCalib") + self.populateStackDetectionCalib() self.setMinMaxObjID(["StackDetection"])