Index: /branches/eam_branches/ipp-20130419/DataStore/lib/DataStore/Utils.pm =================================================================== --- /branches/eam_branches/ipp-20130419/DataStore/lib/DataStore/Utils.pm (revision 35553) +++ /branches/eam_branches/ipp-20130419/DataStore/lib/DataStore/Utils.pm (revision 35554) @@ -44,5 +44,5 @@ %KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits IPP-MOPS IPP-PSPS ipp-psps notset ); %KNOWN_FILESET_TYPES = map { $_ => 1 } qw( OBJECT BIAS DARK SKYFLAT DOMEFLAT OOF SHACKHARTMANN PSREQUEST PSRESULTS IPP-MOPS XRAY FOCUS MOPS_DETECTABILITY_QUERY MOPS_DETECTABILITY_RESPONSE MOPS_TRANSIENT_DETECTIONS LED notset IPP_PSPS IPP-DIST table); -%KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ipp-dist ipp-misc dqresults); +%KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ipp-dist ipp-misc dqresults IPP-MOPS IPP-PSPS PSRESULTS); =pod Index: /branches/eam_branches/ipp-20130419/doc/stamps/stampsICD.tex =================================================================== --- /branches/eam_branches/ipp-20130419/doc/stamps/stampsICD.tex (revision 35553) +++ /branches/eam_branches/ipp-20130419/doc/stamps/stampsICD.tex (revision 35554) @@ -9,5 +9,5 @@ \project{Pan-STARRS IPP} \organization{Institute for Astronomy} -\version{1.0} +\version{2.0} \docnumber{PSDC-940-011-00} @@ -48,9 +48,12 @@ one or more FITS binary tables called \emph{request tables}. Each row in a request table contains a postage stamp request specification. -Each request specification causes zero or more postage stamp jobs to be processed. +Each request specification causes zero or more postage stamp jobs to be created. Images are added to the output file set for each successful job. The request specification give the parameters that the PSS uses to determine the images of interest and the Region of Interest. + +Note: The value "null" for strings or 0 for numbers may be used to disable the application of +certain parameters. \subsection{Request Table Contents} @@ -69,6 +72,8 @@ \hline \code{EXTNAME} & string & \code{PS1_PS_REQUEST} \\ -\code{EXTVER} & string & 1 \\ +\code{EXTVER} & string & 2 \\ \code{REQ_NAME} & string & unique name of the request (see text) \\ +\code{ACTION} & string & PROCESS \\ +\code{EMAIL} & string & Email address of submitting user\\ \hline \end{tabular} @@ -81,9 +86,10 @@ This value is used for the name of the request's output file set. -The postage stamp server supports three types of "postage stamp" job specified by the value + +The postage stamp server supports two types of "postage stamp" job specified by the value the column \code{JOB_TYPE}. If the value is \emph{stamp} postage stamp images will be extracted from the images of interest. If the value of \code{JOB_TYPE} is \emph{get\_image} -the data from the selected images' "runs" with (satellite streaks removed) are bundled -in and placed on the output file set. +the data from the selected images' "runs" are bundled in the format used by the IPP distribution system +and are placed on the output file set. \begin{table}[h] @@ -97,4 +103,6 @@ \code{ROWNUM} & U32 & & must be unique in file\\ \code{PROJECT} & string & gpc1 & \\ +\code{SURVEY_NAME} & string & PS1 survey names & see table 3 \\ +\code{IPP_RELEASE} & string & IPP release name & e.g. 3PI.PV1, 3PI.nightly \\ \code{JOB_TYPE} & string & stamp or get\_image & \\ \\ @@ -117,6 +125,9 @@ \code{MJD_MIN} & F64 & Modified Julian Day & \\ \code{MJD_MAX} & F64 & & \\ -\\ -\code{COMMENT} & string & copied to the results flie & \\ +\code{RUN_TYPE} & string & stack or diff type & deep, reference, nightly, warp-stack, etc.\\ +\code{FWHM_MIN} & F64 & minimum measured seeing (pixels) & \\ +\code{FWHM_MAX} & F64 & maximum measured seeing (pixels) & \\ +\\ +\code{COMMENT} & string & copied to the results file & \\ \hline \end{tabular} @@ -159,5 +170,5 @@ \end{itemize} -Note that at this time the REQ\_TYPE byexp is not implemented for IMG\_TYPE stack. +Note that REQ\_TYPE byexp is not implemented for IMG\_TYPE stack. COMPONENT must also be supplied for request specifications where all of the following are true @@ -173,18 +184,67 @@ \end{itemize} -The special value 'all' may be used to select all component images but this is not recommended. +The special value "all" may be used to select all component components (chips or skycells) of a run. + \subsubsection{Image Selection Cuts}\label{ImageCuts} -A request specification may refer to many images. The scope of the search can be -limited to certain dates with the parameters MJD\_MIN and MJD\_MAX. The specified range is inclusive. -Note: these cuts have no effect for stack stage requests. +A request specification may refer to many images. The scope of the search can be reduced by providing +the selection parameters. + +SURVEY\_NAME (new in version 2) restricts the queries to images for a given PS1 survey. Valid values are +shown in the table below. The value "null" may also be used. + +\begin{table}[h] +\begin{center} +\caption{PS1 SURVEYNAME Values}\label{surveys} +\begin{tabular}{ll} +\hline +\hline +{\bf SURVEY\_NAME} & {\bf Description } \\ +\hline +3PI & PS1 3PI Survey \\ +MD01 & PS1 MD01 XMM-LSS-DXS 022224-043000 \\ +MD02 & PS1 MD02 CDFS/GOODS/GEMS 033224-274800 \\ +MD03 & PS1 MD03 IFA/Lynx 084300+444000 \\ +MD04 & PS1 MD04 COSMOS 100000+021200 \\ +MD05 & PS1 MD05 Lockman-DXS 110000+570000 \\ +MD06 & PS1 MD06 NGC 4258 121857+471814 \\ +MD07 & PS1 MD07 VISTA-Video1 140000+050000 \\ +MD08 & PS1 MD08 EliasN1-DXS 160000+570000 \\ +MD09 & PS1 MD09 Vimos4-DXS-SSA 220000+003000 \\ +MD10 & PS1 MD10 Deep2 233000+000000 \\ +M31 & PS1 M31 Andromeda 004242+411600 \\ +STS1 & PS1 STS1 Outskirts of Bulge 195048+170339 \\ +STS2 & PS1 STS2 Hyades 004000+150000 \\ +STS3 & PS1 STS3 Praesepe 083000+200000 \\ +SSS & PS1 Solar System Survey \\ +\hline +\end{tabular} +\end{center} +\end{table} + +IPP\_RELEASE (new in version 2) restricts the search for data from a particular "release" of IPP data. +If "null" the system choses data from the highest priority release ("best") of the given exposure or skycell. DATA\_GROUP restricts the search to runs with the given value. Note that the SQL database lookups -are done with a "LIKE" comparison so SQL wild cards may be used. +are done with a "LIKE" comparison so SQL wild cards may be used. With Version 2 of this specification +use of DATA\_GROUP is deprecated. It is ignored if a non null value for SURVEY\_NAME or IPP\_RELEASE is +supplied. + +Searches may be limited to certain dates with the parameters MJD\_MIN and MJD\_MAX. The specified range is inclusive. +Note: For stack stage requests, these cuts will have no effect on queries for deep or reference stacks. The REQFILT parameter may be used to restrict the search for images taken with a given filter. -If the filter is a single character (g, r, i, z, y) the server appends a '\%' character to the -sql query. +If the filter is a single character (g, r, i, z, y, w) the server appends a '\%' character to the +sql query. If "null" data from all filters is queried. + +The optional parameter RUN\_TYPE is used for specify the type of stack or difference images to consider as +images of interest. + +For difference images valid values are warp\_warp warp\_stack and stack\_stack. + +For stacks valid values are "deep", "reference", "nightly", and "notnightly". For stacks, this selector +is only applied if SURVEY\_NAME or IPP\_RELEASE is supplied. + @@ -230,4 +290,7 @@ When pixel coordinates are used the coordinate system is that of the image. (For chip level images, before the stamps are created the cells are mosaicked into a single image.) + +Note: If the width and height are both equal to zero, the entire input image is included in the stamp. Otherwise +values less than or equal to zero will generate an error. \subsection{Request Options} @@ -252,5 +315,5 @@ PSTAMP\_SELECT\_INVERSE & 1024 & make stamps from inverse difference images (applies only for diff) \\ PSTAMP\_SELECT\_UNCONV & 2048 & make stamps from unconvolved stack images (applies only for stack) \\ -PSTAMP\_RESTORE\_BACKGROUND & 4096 & (TODO) restore background correction to chip stage images before making stamp \\ +PSTAMP\_RESTORE\_BACKGROUND & 4096 & restore background correction to chip stage images before making stamp \\ PSTAMP\_USE\_IMFILE\_ID & 16384 & for byid requests select images using image id instead of stage id \\ PSTAMP\_NO\_WAIT\_FOR\_UPDATE & 32768 & if images are not available because they have been cleaned don't wait \\ @@ -299,5 +362,6 @@ the pixels for requested images, if possible. This is referred to as the update process. If the user does not wish to wait for the updates to complete he should include -\code{PSTAMP_NO_WAIT_FOR_UPDATE} in the \code{OPTION_MASK}. +\code{PSTAMP_NO_WAIT_FOR_UPDATE} in the \code{OPTION_MASK}. In this case the row will receive the error +code PSTAMP\_NOT\_AVAIALABLE. @@ -353,6 +417,10 @@ \code{EXPTIME} & F64 & exposure time of parent image \\ \code{FPA_ID} & string & orginal FPA\_ID used at injection \\ -\\ -\code{PROJECT} & string & Values from initial request specification\\ +\hline +& Values from corresponding row in the request file\\ +\hline +\code{PROJECT} & string & \\ +\code{SURVEY_NAME} & string & \\ +\code{IPP_RELEASE} & string & \\ \code{JOB_TYPE} & string & \\ \\ @@ -372,4 +440,7 @@ \code{MJD_MAX} & F64 & \\ \code{DATA_GROUP} & string & \\ +\code{RUN_TYPE} & string & \\ +\code{FWHM_MIN} & F64 & \\ +\code{FWHM_MAX} & F64 & \\ \code{COMMENT} & string & \\ \\ @@ -393,8 +464,8 @@ have the name \code{ROWNUM_I.mk.fits} -JOB\_ID is the value of the postage stamp server's internal job id. (This is provided primarily as a debugging +JOB\_ID is the value of the postage stamp server's internal job id. aid.) -The column named PROJECT and following columns are copies from the input request specification. +The column named PROJECT and following columns are copied from the input request specification. If ERROR\_CODE is non-zero an error occurred when processing the request specification or the job. @@ -419,9 +490,10 @@ PSTAMP\_UNKNOWN\_PRODUCT & 22 & Unknown product in request specification \\ PSTAMP\_NO\_IMAGE\_MATCH & 23 & No images matched the request \\ -PSTAMP\_NOT\_DESTREAKED & 24 & Image matched, but not yet de-streaked \\ +PSTAMP\_NOT\_DESTREAKED & 24 & Obsolete - No longer used\\ PSTAMP\_NOT\_AVAILABLE & 25 & Image not available (temporary) \\ PSTAMP\_GONE & 26 & Image is no longer available \\ PSTAMP\_NO\_JOBS\_QUEUED & 27 & Request specification yielded no jobs. See parse\_error.txt \\ PSTAMP\_NO\_OVERLAP & 28 & Center coordinates are not contained in any image of interest \\ +PSTAMP\_NOT\_AUTHORIZED & 29 & user is not allowed access to the selected data\\ \hline Index: /branches/eam_branches/ipp-20130419/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- /branches/eam_branches/ipp-20130419/ippToPsps/config/ippToPspsDbSchema.sql (revision 35553) +++ /branches/eam_branches/ipp-20130419/ippToPsps/config/ippToPspsDbSchema.sql (revision 35554) @@ -48,5 +48,5 @@ `purged` tinyint(4) NOT NULL default '0', PRIMARY KEY (`batch_id`) -) ENGINE=InnoDB AUTO_INCREMENT=328286 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1; SET character_set_client = @saved_cs_client; @@ -69,5 +69,5 @@ KEY `fk_skychunk` (`skychunk`), CONSTRAINT `fk_skychunk` FOREIGN KEY (`skychunk`) REFERENCES `skychunk` (`name`) ON DELETE CASCADE -) ENGINE=InnoDB AUTO_INCREMENT=44588 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1; SET character_set_client = @saved_cs_client; @@ -91,5 +91,5 @@ PRIMARY KEY (`id`), UNIQUE KEY `unique_key` (`host`,`pid`) -) ENGINE=InnoDB AUTO_INCREMENT=1227 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1; SET character_set_client = @saved_cs_client; Index: /branches/eam_branches/ipp-20130419/ippToPsps/config/settings.xml =================================================================== --- /branches/eam_branches/ipp-20130419/ippToPsps/config/settings.xml (revision 35553) +++ /branches/eam_branches/ipp-20130419/ippToPsps/config/settings.xml (revision 35554) @@ -45,4 +45,137 @@ 16 + + ipptopsps_scratch + ipp045 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp046 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp047 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp048 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp049 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp050 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp051 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp052 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp053 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp054 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp055 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp056 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp057 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp058 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp059 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp060 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp061 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp062 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp063 + dvo + dvo + 32 + ipptopsps_scratch @@ -52,4 +185,58 @@ 32 + + ipptopsps_scratch + ipp065 + dvo + dvo + 32 + + + ipptopsps_scratch + ipp066 + dvo + dvo + 32 + + + ipptopsps_scratch + stare00 + dvo + dvo + 64 + + + ipptopsps_scratch + stare01 + dvo + dvo + 64 + + + ipptopsps_scratch + stare02 + dvo + dvo + 64 + + + ipptopsps_scratch + stare03 + dvo + dvo + 64 + + + ipptopsps_scratch + stare04 + dvo + dvo + 64 + + + + + + ipptopsps_scratch Index: /branches/eam_branches/ipp-20130419/ippToPsps/doc/notes.txt =================================================================== --- /branches/eam_branches/ipp-20130419/ippToPsps/doc/notes.txt (revision 35553) +++ /branches/eam_branches/ipp-20130419/ippToPsps/doc/notes.txt (revision 35554) @@ -1,2 +1,41 @@ + +2013.04.27 + + need to move the conversion between mags and flux into dvopsps. + + * detection table + * psfFlux : uses PSF_INST_MAG instead of PSF_INST_FLUX + * psfFluxErr : uses PSF_INST_MAG_SIG and PSF_INST_MAG instead of PSF_INST_FLUX_SIG + * apFlux : uses AP_MAG instead of AP_FLUX + * kronFlux : correctly uses KRON_FLUX + * kronFluxErr : correctly uses KRON_FLUX_ERR + + ** psfFlux is forced to 0.0000001 instead of 0.0 -- why? + + * stack detection table (same issues as detection table) + * psfFlux : uses PSF_INST_MAG instead of PSF_INST_FLUX + * psfFluxErr : uses PSF_INST_MAG_SIG and PSF_INST_MAG instead of PSF_INST_FLUX_SIG + * apFlux : uses AP_MAG instead of AP_FLUX + * kronFlux : correctly uses KRON_FLUX + * kronFluxErr : correctly uses KRON_FLUX_ERR + + * psfFlux is set to 2 sigma if psfFlux is NULL + + * object + * Mean PSF Mag : direct from DVO MAG + * Mean PSF Mag Err : direct from DVO MAG_ERR + * Mean Kron Mag : direct from DVO KRON MAG + * Mean Kron Mag Err : direct from DVO KRON MAG_ERR + + * Stack PSF Mag : from DVO stack psf flux + * Stack PSF Mag Err : from DVO stack psf flux err + * Stack Kron Mag : from DVO stack kron flux + * Stack Kron Mag Err : from DVO stack kron flux err + + + + + +old notes: example of joining ipptopsps and gpc1 tables: Index: /branches/eam_branches/ipp-20130419/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20130419/ippToPsps/jython/batch.py (revision 35553) +++ /branches/eam_branches/ipp-20130419/ippToPsps/jython/batch.py (revision 35554) @@ -65,4 +65,5 @@ self.scratchDb = scratchDb self.batchType = batchType; + self.skipBatch = False; # set to true if there is a non-fatal error self.pspsVoTableFilePath = self.config.configDir + "tables." + batchType + ".vot" Index: /branches/eam_branches/ipp-20130419/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/eam_branches/ipp-20130419/ippToPsps/jython/detectionbatch.py (revision 35553) +++ /branches/eam_branches/ipp-20130419/ippToPsps/jython/detectionbatch.py (revision 35554) @@ -439,6 +439,6 @@ ,X_PSF_SIG \ ,Y_PSF_SIG \ - ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + " \ - ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + ")) / 1.085736) \ + ,PSF_INST_FLUX / " + self.header['EXPTIME'] + " \ + ,PSF_INST_FLUX_SIG / " + self.header['EXPTIME'] + " \ ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + self.header['EXPTIME'] + " \ ,PSF_MAJOR \ @@ -450,5 +450,5 @@ ,MOMENTS_XY \ ,MOMENTS_YY \ - ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + " \ + ,AP_FLUX / " + self.header['EXPTIME'] + " \ ,KRON_FLUX / " + self.header['EXPTIME'] + " \ ,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \ @@ -466,11 +466,30 @@ # self.logger.info(sql) - self.scratchDb.execute(sql) - +# these were used above (left over from before PSF_INST_FLUX, PSF_INST_FLUX_SIG were available? +# ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + " +# ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + ")) / 1.085736) +# ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + " + + self.scratchDb.execute(sql) + + # XXX EAM : I removed this old fix for invalid fluxes # add a instFlux = 0.0 -> 0.000001 - # XXX EAM : why is this done? - sql="UPDATE " + pspsTableName + " SET psfFlux = 0.0000001 WHERE psfFlux = 0" - self.scratchDb.execute(sql) + # XXX EAM : why is this done? + # sql="UPDATE " + pspsTableName + " SET psfFlux = 0.0000001 WHERE psfFlux = 0" + # self.scratchDb.execute(sql) + # NOTE : Flux limits : in the current PSPS schema, negative fluxes + # cause problems for sql queries which work in mags as SQL cannot do + # something like (f < 0.0) ? -999 : -2.5*log10(f) + # as a result, the negative fluxes here result in floating point errors + sql = "UPDATE " + pspsTableName + " SET psfFlux = 1e20 WHERE psfFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE " + pspsTableName + " SET apFlux = 1e20 WHERE apFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE " + pspsTableName + " SET kronFlux = 1e20 WHERE kronFlux <= 0.0" + self.scratchDb.execute(sql) + # we don't delete these anymore results['NULLINSTFLUX'] = 0; @@ -1016,5 +1035,5 @@ # XXX EAM NOTE : this is fragile : requires PS1_V4 - columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" + columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG AP_FLUX KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" return super(DetectionBatch, self).importIppTables(columns, regex) Index: /branches/eam_branches/ipp-20130419/ippToPsps/jython/objectbatch.py =================================================================== --- /branches/eam_branches/ipp-20130419/ippToPsps/jython/objectbatch.py (revision 35553) +++ /branches/eam_branches/ipp-20130419/ippToPsps/jython/objectbatch.py (revision 35554) @@ -137,12 +137,18 @@ ," + filter[1] + "MeanKronMag = MAG_KRON \ ," + filter[1] + "MeanKronMagErr = MAG_KRON_ERR \ - ," + filter[1] + "StackPSFMag = -2.5 * log10(FLUX_PSF) + 8.9 \ - ," + filter[1] + "StackPSFMagErr = FLUX_PSF_ERR / FLUX_PSF \ - ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) + 8.9 \ - ," + filter[1] + "StackKronMagErr = FLUX_KRON_ERR / FLUX_KRON \ + ," + filter[1] + "StackPSFMag = STACK_PSF_MAG \ + ," + filter[1] + "StackPSFMagErr = STACK_PSF_MAG_ERR \ + ," + filter[1] + "StackKronMag = STACK_KRON_MAG \ + ," + filter[1] + "StackKronMagErr = STACK_KRON_MAG_ERR \ ," + filter[1] + "20pct = MAG_20/1000 \ ," + filter[1] + "80pct = MAG_80/1000 \ ," + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) \ ," + filter[1] + "StackDetectID = STACK_DETECT_ID" + +# old method for setting the Stack (PSF,Kron) Mags: +# ," + filter[1] + "StackPSFMag = -2.5 * log10(FLUX_PSF) + 8.9 \ +# ," + filter[1] + "StackPSFMagErr = FLUX_PSF_ERR / FLUX_PSF \ +# ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) + 8.9 \ +# ," + filter[1] + "StackKronMagErr = FLUX_KRON_ERR / FLUX_KRON \ #self.logger.info(sql) @@ -216,24 +222,22 @@ cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps") - self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON_ERR") - sql = "UPDATE " + cpsTableName + " set FLUX_KRON_ERR = NULL where FLUX_KRON_ERR < 1e-37 AND FLUX_KRON_ERR > -1e-37 " + if False: + self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON_ERR") + sql = "UPDATE " + cpsTableName + " set FLUX_KRON_ERR = NULL where FLUX_KRON_ERR < 1e-37 AND FLUX_KRON_ERR > -1e-37 " - try: - self.scratchDb.execute(sql) - except: - self.logger.errorPair("Couldn't squash out of range stuff", sql) - return False - - self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON") - sql = "UPDATE " + cpsTableName + " set FLUX_KRON = NULL where FLUX_KRON < 1e-37 AND FLUX_KRON > -1e-37 " + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't squash out of range stuff", sql) + return False + + self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON") + sql = "UPDATE " + cpsTableName + " set FLUX_KRON = NULL where FLUX_KRON < 1e-37 AND FLUX_KRON > -1e-37 " - try: - self.scratchDb.execute(sql) - except: - self.logger.errorPair("Couldn't squash out of range stuff", sql) - return False - - - + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't squash out of range stuff", sql) + return False Index: /branches/eam_branches/ipp-20130419/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/eam_branches/ipp-20130419/ippToPsps/jython/stackbatch.py (revision 35553) +++ /branches/eam_branches/ipp-20130419/ippToPsps/jython/stackbatch.py (revision 35554) @@ -482,6 +482,6 @@ ,X_PSF_SIG \ ,Y_PSF_SIG \ - ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + " \ - ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + ")) / 1.085736) \ + ,PSF_INST_FLUX / " + str(self.expTime) + " \ + ,PSF_INST_FLUX_SIG / " + str(self.expTime) + " \ ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + str(self.expTime) + " \ ,SKY / " + str(self.expTime) + " \ @@ -504,5 +504,5 @@ ,MOMENTS_R1 \ ,MOMENTS_RH \ - ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + " \ + ,AP_FLUX / " + str(self.expTime) + " \ ,NULL \ ,KRON_FLUX / " + str(self.expTime) + " \ @@ -515,4 +515,9 @@ FROM SkyChip_psf" +# these were used above (left over from before PSF_INST_FLUX, PSF_INST_FLUX_SIG were available? +# ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + " +# ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + ")) / 1.085736) +# ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + " + # print "sql: ", sql # response = raw_input("ready to insert stack det ") @@ -526,4 +531,8 @@ self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0") + # XXX EAM : this seems quite inefficient : these commands use updates to set + # fields which are constant across all rows. Why not just include these in + # the original insert above? + self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID)) @@ -560,5 +569,6 @@ # response = raw_input("add primary key? ") - if self.stackType == "DEEP_STACK": + ### XXX this should not be a DEEP_STACK only case... + if False and self.stackType == "DEEP_STACK": #if deep stack and instFlux = null and err not null @@ -572,4 +582,17 @@ # response = raw_input("add psf flux ") + # NOTE : Flux limits : in the current PSPS schema, negative fluxes + # cause problems for sql queries which work in mags as SQL cannot do + # something like (f < 0.0) ? -999 : -2.5*log10(f) + # as a result, the negative fluxes here result in floating point errors + sql = "UPDATE StackDetection SET psfFlux = 1e20 WHERE psfFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackDetection SET apFlux = 1e20 WHERE apFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackDetection SET kronFlux = 1e20 WHERE kronFlux <= 0.0" + self.scratchDb.execute(sql) + #leave null instflux in #self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux") @@ -930,4 +953,6 @@ def populatePspsTables(self): + self.skipBatch = False + if not self.useFullTables: if not self.getIDsFromDVO(): Index: /branches/eam_branches/ipp-20130419/ippToPsps/test/fulltest.sh =================================================================== --- /branches/eam_branches/ipp-20130419/ippToPsps/test/fulltest.sh (revision 35553) +++ /branches/eam_branches/ipp-20130419/ippToPsps/test/fulltest.sh (revision 35554) @@ -1,3 +1,5 @@ #!/bin/csh -f + +chmod +x *.dvo *.sh # a bit crude : get the datadir path by interpollation of the PATH element for psconfig @@ -23,8 +25,28 @@ set objectbatch = 1 -set args = "" - -while ("$1" != "") - switch ("$1") +set targ = ($argv) +set args = "" + +while ($#targ) + switch ("$targ[1]") + case -h + case -help + echo "USAGE: fulltest.sh [-XX] [+XX]" + echo " -XX : turn off features" + echo " +XX : turn on features" + echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, object" + exit 2 + case -all + set mkgpc1 = 0 + set initdb = 0 + set initscratch = 0 + set initbatch = 0 + set camqueue = 0 + set cambatch = 0 + set stackqueue = 0 + set stackbatch = 0 + set objectqueue = 0 + set objectbatch = 0 + breaksw; case -mkgpc1 set mkgpc1 = 0 @@ -52,9 +74,65 @@ breaksw; default: - set args=($args $1); + set args=($args $targ[1]); breaksw; endsw - shift + shift targ end + +set targ = ($args) +set args = "" + +while ($#targ) + switch ("$targ[1]") + case +all + set mkgpc1 = 1 + set initdb = 1 + set initscratch = 1 + set initbatch = 1 + set camqueue = 1 + set cambatch = 1 + set stackqueue = 1 + set stackbatch = 1 + set objectqueue = 1 + set objectbatch = 1 + breaksw; + case +mkgpc1 + set mkgpc1 = 1 + breaksw; + case +initdb + set initdb = 1 + breaksw; + case +initscratch + set initscratch = 1 + breaksw; + case +initbatch + set initbatch = 1 + breaksw; + case "+camera" + set camqueue = 1 + set cambatch = 1 + breaksw; + case +stack + set stackqueue = 1 + set stackbatch = 1 + breaksw; + case +object + set objectqueue = 1 + set objectbatch = 1 + breaksw; + default: + set args=($args $targ[1]); + breaksw; + endsw + shift targ +end + +if ($args[1] != "") then + echo "USAGE: fulltest.sh [-XX] [+XX]" + echo " -XX : turn off features" + echo " +XX : turn on features" + echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, object" + exit 2 +endif set stackqueuename = 0 Index: /branches/eam_branches/ipp-20130419/ippToPsps/test/mkgpc1data.dvo =================================================================== --- /branches/eam_branches/ipp-20130419/ippToPsps/test/mkgpc1data.dvo (revision 35553) +++ /branches/eam_branches/ipp-20130419/ippToPsps/test/mkgpc1data.dvo (revision 35554) @@ -37,6 +37,8 @@ macro mkfull init.db - mkcatdir.cam PS1_V3 PS1_V4 - mkcatdir.stk PS1_V3 PS1_V4 + # mkcatdir.cam PS1_V3 PS1_V4 + # mkcatdir.stk PS1_V3 PS1_V4 + mkcatdir.cam PS1_V4 PS1_V4 + mkcatdir.stk PS1_V4 PS1_V4 insert.stack.set end