Index: branches/pap/dbconfig/background.md =================================================================== --- branches/pap/dbconfig/background.md (revision 28179) +++ branches/pap/dbconfig/background.md (revision 28179) @@ -0,0 +1,70 @@ +# Tables to support background replacement + +# Background replacement on a chipRun +chipBackgroundRun METADATA + chip_bg_id S64 0 + chip_id S64 0 + state STR 64 + workdir STR 255 + label STR 64 + data_group STR 64 + dist_group STR 64 + reduction STR 64 + note STR 255 + registered TAI NULL +END + +# Results of background replacement from chipBackgroundRun +chipBackgroundImfile METADATA + chip_bg_id S64 0 + class_id STR 64 + path_base STR 255 + magicked S64 0 + dtime_script F32 0.0 + hostname STR 64 + quality S16 0 + fault S16 0 + software_ver STR 16 + bg F64 0.0 + bg_stdev F64 0.0 + maskfrac_npix F32 0.0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 +END + +# Background replacement on a warpRun (utilising chipBackgroundRun) +warpBackgroundRun METADATA + warp_bg_id S64 0 + warp_id S64 0 + chip_bg_id S64 0 + state STR 64 + workdir STR 255 + label STR 64 + data_group STR 64 + dist_group STR 64 + reduction STR 64 + note STR 255 + registered TAI NULL +END + +# Results of background replacement from warpBackgroundRun +warpBackgroundSkyfile METADATA + warp_bg_id S64 0 + skycell_id STR 64 + path_base STR 255 + magicked S64 0 + dtime_script F32 0.0 + hostname STR 64 + quality S16 0 + fault S16 0 + software_ver STR 16 + bg F64 0.0 + bg_stdev F64 0.0 + maskfrac_npix F32 0.0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 +END Index: branches/pap/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 28176) +++ branches/pap/ippTools/share/pxadmin_create_tables.sql (revision 28179) @@ -994,5 +994,5 @@ tess_id VARCHAR(64), filter VARCHAR(64), - software_ver VARCHAR(16), + software_ver VARCHAR(16), note VARCHAR(255), PRIMARY KEY(stack_id), @@ -1042,5 +1042,5 @@ quality SMALLINT NOT NULL DEFAULT 0, fault SMALLINT, - software_ver VARCHAR(16), + software_ver VARCHAR(16), PRIMARY KEY(stack_id), KEY(dtime_stack), @@ -1065,5 +1065,5 @@ exposure TINYINT DEFAULT 0, magicked BIGINT, - software_ver VARCHAR(16), + software_ver VARCHAR(16), maskfrac_npix FLOAT, maskfrac_static FLOAT, @@ -1134,9 +1134,9 @@ fault SMALLINT, magicked BIGINT, - software_ver VARCHAR(16), - maskfrac_npix INT, - maskfrac_static FLOAT, - maskfrac_dynamic FLOAT, - maskfrac_magic FLOAT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, maskfrac_advisory FLOAT, PRIMARY KEY(diff_id, skycell_id), @@ -1667,4 +1667,6 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Processing labels with their priorities CREATE TABLE Label ( label VARCHAR(64), @@ -1677,4 +1679,102 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Tables to support background replacement + +-- Background replacement on a chipRun +CREATE TABLE chipBackgroundRun ( + chip_bg_id BIGINT AUTO_INCREMENT, -- unique identifier + chip_id BIGINT NOT NULL, -- link to chipRun + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(chip_bg_id), + KEY(chip_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group) + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Results of background replacement from chipBackgroundRun +CREATE TABLE chipBackgroundImfile ( + chip_bg_id BIGINT NOT NULL, -- unique identifier + class_id VARCHAR(64) NOT NULL, -- class (component) identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + magicked BIGINT, -- magic_id if magicked + dtime_script FLOAT, -- elapsed time for script + hostname VARCHAR(64) NOT NULL, -- host that executed script + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + software_ver VARCHAR(16), -- software version + bg FLOAT, -- background level + bg_stdev FLOAT, -- stdev of background + maskfrac_npix FLOAT, -- Number of pixels masked + maskfrac_static FLOAT, -- Fraction masked static + maskfrac_dynamic FLOAT, -- Fraction masked dynamic + maskfrac_magic FLOAT, -- Fraction masked magic + maskfrac_advisory FLOAT, -- Fraction masked advisory + PRIMARY KEY(chip_bg_id,class_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Background replacement on a warpRun (utilising chipBackgroundRun) +CREATE TABLE warpBackgroundRun ( + warp_bg_id BIGINT AUTO_INCREMENT, -- unique identifier + warp_id BIGINT NOT NULL, -- link to warpRun + chip_bg_id BIGINT NOT NULL, -- link to chipBackgroundRun + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(warp_bg_id), + KEY(warp_id), + KEY(chip_bg_id) + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group) + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id), + FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Results of background replacement from warpBackgroundRun +CREATE TABLE warpBackgroundSkyfile ( + warp_bg_id BIGINT NOT NULL, -- unique identifier + skycell_id VARCHAR(64) NOT NULL, -- skycell identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + magicked BIGINT, -- magic_id if magicked + dtime_script FLOAT, -- elapsed time for script + hostname VARCHAR(64) NOT NULL, -- host that executed script + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + software_ver VARCHAR(16), -- software version + bg FLOAT, -- background level + bg_stdev FLOAT, -- stdev of background + maskfrac_npix FLOAT, -- Number of pixels masked + maskfrac_static FLOAT, -- Fraction masked static + maskfrac_dynamic FLOAT, -- Fraction masked dynamic + maskfrac_magic FLOAT, -- Fraction masked magic + maskfrac_advisory FLOAT, -- Fraction masked advisory + PRIMARY KEY(warp_bg_id,skycell_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(warp_bg_id) REFERENCES warpBackgroundRun(warp_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + + -- This comment line is here to avoid empty query error. -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy. Index: branches/pap/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_drop_tables.sql (revision 28176) +++ branches/pap/ippTools/share/pxadmin_drop_tables.sql (revision 28179) @@ -86,4 +86,8 @@ DROP TABLE IF EXISTS Label; DROP TABLE IF EXISTS pstampWebRequest; +DROP TABLE IF EXISTS chipBackgroundRun; +DROP TABLE IF EXISTS chipBackgroundImfile; +DROP TABLE IF EXISTS warpBackgroundRun; +DROP TABLE IF EXISTS warpBackgroundSkyfile; SET FOREIGN_KEY_CHECKS=1 Index: branches/pap/psModules/src/detrend/pmGainTweak.c =================================================================== --- branches/pap/psModules/src/detrend/pmGainTweak.c (revision 28179) +++ branches/pap/psModules/src/detrend/pmGainTweak.c (revision 28179) @@ -0,0 +1,59 @@ +#include +#include + +#include "pmFPA.h" + +#include "pmGainTweak.h" + +bool pmGainTweak(const pmCell *cell) +{ + PS_ASSERT_PTR_NON_NULL(cell, false); + pmChip *chip = cell->parent; + PS_ASSERT_PTR_NON_NULL(chip, false); + + int numCells = chip->cells; // Number of cells + psVector *gains = psVectorAlloc(numCells, PS_TYPE_F32); // Gain for each cell + psVector *mask = psVectorAlloc(numCells, PS_TYPE_VECTOR_MASK); // Mask for gains + psVectorInit(mask, 0); + int numGood = 0; // Number of good gains + + for (int i = 0; i < numCells; i++) { + pmCell *otherCell = chip->cells->data[i]; // A different cell + if (otherCell == cell) { + mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0xFF; + continue; + } + float gain = psMetadataLookupF32(NULL, otherCell->concepts, "CELL.GAIN"); // Gain for cell + gains->data.F32[i] = gain; + if (!isfinite(gains->data.F32[i])) { + mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0xFF; + } else { + numGood++; + } + } + + if (numGood == 0) { + psError(PM_ERR_DETREND, true, "No other cell gains available with which to tweak gain."); + psFree(gains); + psFree(mask); + return false; + } + + psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN); // Statistics + if (!psVectorStats(stats, gains, NULL, mask, 0xFF)) { + psError(psErrorCodeLast(), false, "Unable to determine target gain"); + psFree(stats); + psFree(gains); + psFree(mask); + return false; + } + float target = stats->sampleMedian; // Target gain + psFree(stats); + psFree(gains); + psFree(mask); + + psMetadataItem *item = psMetadataLookup(cell->concepts, "CELL.GAIN"); + item->data.F32 = target; + + return true; +}