Index: /branches/faketool/ippTools/scripts/regtest.sh =================================================================== --- /branches/faketool/ippTools/scripts/regtest.sh (revision 17937) +++ /branches/faketool/ippTools/scripts/regtest.sh (revision 17938) @@ -33,5 +33,5 @@ $regtool -pendingexp|| exit 1 -$regtool -addprocessedexp -exp_id $exp_id1 -exp_name t10 -inst gpc -telescope ps1 -exp_tag batman.t10 -filelevel OTA -filter r -airmass 10 -ra 1 -decl 2 -exp_type bias -exp_time 0 -bg 10 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 45 -posang 10 -object dog -dateobs "2006-10-20T10:10:10Z" -label foobar $* || exit 1 +$regtool -addprocessedexp -exp_id $exp_id1 -exp_name t10 -inst gpc -telescope ps1 -exp_tag batman.t10 -filelevel OTA -filter r -airmass 10 -ra 1 -decl 2 -exp_type bias -exp_time 0 -bg 10 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 45 -posang 10 -object dog -dateobs "2006-10-20T10:10:10Z" -label foobar -tess_id moldyshoe $* || exit 1 $regtool -pendingimfile || exit 1 @@ -43,3 +43,3 @@ $regtool -pendingexp|| exit 1 -$regtool -addprocessedexp -exp_id $exp_id2 -exp_name t11 -inst gpc -telescope ps1 -exp_tag batman.t11 -filelevel OTA -filter r -airmass 11 -ra 1 -decl 2 -exp_type bias -exp_time 0 -bg 11 -bg_stdev 1 -bg_mean_stdev 11 -alt 11 -az 11 -ccd_temp 45 -posang 11 -object dog -dateobs 2006-10-20T10:10:10Z -label foobar $* || exit 1 +$regtool -addprocessedexp -exp_id $exp_id2 -exp_name t11 -inst gpc -telescope ps1 -exp_tag batman.t11 -filelevel OTA -filter r -airmass 11 -ra 1 -decl 2 -exp_type bias -exp_time 0 -bg 11 -bg_stdev 1 -bg_mean_stdev 11 -alt 11 -az 11 -ccd_temp 45 -posang 11 -object dog -dateobs 2006-10-20T10:10:10Z -label foobar -tess_id moldyshoe $* || exit 1 Index: /branches/faketool/ippTools/scripts/warptest.sh =================================================================== --- /branches/faketool/ippTools/scripts/warptest.sh (revision 17937) +++ /branches/faketool/ippTools/scripts/warptest.sh (revision 17938) @@ -3,5 +3,5 @@ set -o verbose -camtest.sh || exit 1 +faketest.sh || exit 1 #warptool -definerun -mode warp -workdir file:///foo || exit 1 @@ -15,6 +15,4 @@ warptool -tooverlap -warp_id 1 || exit 1 - -exit 1 warptool -addoverlap -mapfile mapfile.txt || exit 1 Index: /branches/faketool/ippTools/share/Makefile.am =================================================================== --- /branches/faketool/ippTools/share/Makefile.am (revision 17937) +++ /branches/faketool/ippTools/share/Makefile.am (revision 17938) @@ -18,4 +18,8 @@ detselect_search.sql \ detselect_select.sql \ + faketool_completely_processed_exp.sql \ + faketool_pendingimfile.sql \ + faketool_processedimfile.sql \ + faketool_queue_cam_id.sql \ dettool_tocorrectimfile.sql \ dettool_tocorrectexp.sql \ Index: /branches/faketool/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/faketool/ippTools/share/pxadmin_create_tables.sql (revision 17937) +++ /branches/faketool/ippTools/share/pxadmin_create_tables.sql (revision 17938) @@ -646,8 +646,73 @@ CREATE TABLE detRegisteredImfile (det_id BIGINT, iteration INT, class_id VARCHAR(64), uri VARCHAR(255), bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, path_base VARCHAR(255), fault SMALLINT NOT NULL, PRIMARY KEY(det_id, iteration, class_id), KEY(fault), FOREIGN KEY (det_id, iteration) REFERENCES detRun(det_id, iteration)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpRun (warp_id BIGINT AUTO_INCREMENT, cam_id BIGINT, mode VARCHAR(64), state VARCHAR(64), workdir VARCHAR(255), workdir_state VARCHAR(64), label VARCHAR(64), dvodb VARCHAR(255), tess_id VARCHAR(64), end_stage VARCHAR(64), registered DATETIME, magiced TINYINT, PRIMARY KEY(warp_id), KEY(warp_id), KEY(cam_id), KEY(mode), KEY(state), KEY(workdir_state), KEY(label), KEY(end_stage), KEY(magiced), INDEX(warp_id, cam_id), FOREIGN KEY (cam_id) REFERENCES camProcessedExp(cam_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpSkyCellMap (warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64), cam_id BIGINT, class_id VARCHAR(64), fault SMALLINT, PRIMARY KEY(warp_id, skycell_id, tess_id, cam_id, class_id), KEY(fault), FOREIGN KEY (warp_id, cam_id) REFERENCES warpRun(warp_id, cam_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpSkyfile (warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64), uri VARCHAR(255), path_base VARCHAR(255), bg DOUBLE, bg_stdev DOUBLE, dtime_warp FLOAT, hostname VARCHAR(64), good_frac FLOAT, xmin INT, xmax INT, ymin INT, ymax INT, ignored TINYINT, fault SMALLINT, PRIMARY KEY(warp_id, skycell_id, tess_id), KEY(good_frac), KEY(ignored), KEY(fault), FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpMask (label VARCHAR(64), PRIMARY KEY(label)) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE warpRun ( + warp_id BIGINT AUTO_INCREMENT, + fake_id BIGINT, + mode VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + registered DATETIME, + magiced TINYINT, + PRIMARY KEY(warp_id), + KEY(warp_id), + KEY(fake_id), + KEY(mode), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(end_stage), + KEY(magiced), + INDEX(warp_id, fake_id), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyCellMap ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + fake_id BIGINT, + class_id VARCHAR(64), + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id, fake_id, class_id), + KEY(fault), + FOREIGN KEY (warp_id, fake_id) + REFERENCES warpRun(warp_id, fake_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyfile ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + uri VARCHAR(255), + path_base VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_warp FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + xmin INT, + xmax INT, + ymin INT, + ymax INT, + ignored TINYINT, + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id), + KEY(good_frac), + KEY(ignored), KEY(fault), + FOREIGN KEY (warp_id, skycell_id, tess_id) + REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE stackRun (stack_id BIGINT AUTO_INCREMENT, state VARCHAR(64), workdir VARCHAR(255), dvodb VARCHAR(255), registered DATETIME, skycell_id VARCHAR(64), tess_id VARCHAR(64), PRIMARY KEY(stack_id), KEY(stack_id), KEY(state), KEY(skycell_id), KEY(tess_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE stackInputSkyfile (stack_id BIGINT, warp_id BIGINT, PRIMARY KEY(stack_id, warp_id), FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id), FOREIGN KEY (warp_id) REFERENCES warpSkyfile(warp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/faketool/ippTools/share/warptool_exp.sql =================================================================== --- /branches/faketool/ippTools/share/warptool_exp.sql (revision 17937) +++ /branches/faketool/ippTools/share/warptool_exp.sql (revision 17938) @@ -1,8 +1,8 @@ SELECT - camProcessedExp.* + fakeRun.* FROM warpRun -JOIN camProcessedExp - USING(cam_id) +JOIN fakeRun + USING(fake_id) WHERE warpRun.state = 'run' - AND camProcessedExp.fault = 0 + AND fakeRun.state = 'stop' Index: /branches/faketool/ippTools/share/warptool_imfile.sql =================================================================== --- /branches/faketool/ippTools/share/warptool_imfile.sql (revision 17937) +++ /branches/faketool/ippTools/share/warptool_imfile.sql (revision 17938) @@ -1,18 +1,26 @@ SELECT DISTINCT rawImfile.*, - warpRun.cam_id, + warpRun.fake_id, chipProcessedImfile.uri as chip_uri, chipProcessedImfile.path_base as chip_path_base, camProcessedExp.path_base as cam_path_base FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + ON camRun.chip_id = chipRun.chip_id JOIN chipProcessedImfile - ON camProcessedExp.chip_id = chipProcessedImfile.chip_id + ON chipRun.chip_id = chipProcessedImfile.chip_id JOIN rawImfile -- is there any reason not to refer back to rawimfiles? - ON chipProcessedImfile.exp_id = rawImfile.exp_id AND chipProcessedImfile.class_id = rawImfile.class_id + ON chipProcessedImfile.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id WHERE warpRun.state = 'run' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' Index: /branches/faketool/ippTools/share/warptool_tooverlap.sql =================================================================== --- /branches/faketool/ippTools/share/warptool_tooverlap.sql (revision 17937) +++ /branches/faketool/ippTools/share/warptool_tooverlap.sql (revision 17938) @@ -1,5 +1,5 @@ SELECT warpRun.warp_id, - warpRun.cam_id, + warpRun.fake_id, warpRun.workdir, warpRun.tess_id, @@ -9,5 +9,7 @@ warpRun.magiced FROM warpRun -JOIN camProcessedExp +JOIN fakeRun + USING(fake_id) +JOIN camRun USING(cam_id) JOIN chipRun @@ -16,11 +18,13 @@ USING(exp_id) LEFT JOIN warpSkyCellMap - USING(warp_id, cam_id) + USING(warp_id, fake_id) LEFT JOIN warpMask ON warpRun.label = warpMask.label WHERE warpRun.state = 'run' - AND camProcessedExp.fault = 0 + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' AND warpSkyCellMap.warp_id IS NULL - AND warpSkyCellMap.cam_id IS NULL + AND warpSkyCellMap.fake_id IS NULL AND warpMask.label IS NULL Index: /branches/faketool/ippTools/src/Makefile.am =================================================================== --- /branches/faketool/ippTools/src/Makefile.am (revision 17937) +++ /branches/faketool/ippTools/src/Makefile.am (revision 17938) @@ -8,7 +8,8 @@ flatcorr \ magictool \ - pstamptool \ + pstamptool \ pxadmin \ pxinject \ + faketool \ pzgetexp \ pzgetimfiles \ @@ -28,4 +29,5 @@ pxconfig.h \ pxdata.h \ + pxfake.h \ pxio.h \ pxregister.h \ @@ -44,7 +46,8 @@ difftool.h \ flatcorr.h \ + faketool.h \ guidetool.h \ magictool.h \ - pstamptool.h \ + pstamptool.h \ pxinject.h \ pzgetexp.h \ @@ -65,4 +68,5 @@ pxdata.c \ pxerrors.c \ + pxfake.c \ pxfault.c \ pxio.c \ @@ -124,4 +128,10 @@ camtoolConfig.c +faketool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +faketool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +faketool_SOURCES = \ + faketool.c \ + faketoolConfig.c + magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) magictool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: /branches/faketool/ippTools/src/camtool.c =================================================================== --- /branches/faketool/ippTools/src/camtool.c (revision 17937) +++ /branches/faketool/ippTools/src/camtool.c (revision 17938) @@ -27,5 +27,4 @@ #include "pxtools.h" -#include "pxwarp.h" #include "pxcam.h" #include "camtool.h" @@ -599,33 +598,33 @@ psFree(output); camProcessedExpRow *row = camProcessedExpRowAlloc( - pendingRow->cam_id, - pendingRow->chip_id, - uri, - bg, - bg_stdev, - bg_mean_stdev, - bias, - bias_stdev, - fringe_0, - fringe_1, - fringe_2, - sigma_ra, - sigma_dec, - ap_resid, - ap_resid_stdev, - zp_mean, - zp_stdev, - fwhm_major, - fwhm_minor, - dtime_detrend, - dtime_photom, - dtime_astrom, - hostname, - n_stars, - n_extended, - n_cr, - n_astrom, - path_base, - code + pendingRow->cam_id, + pendingRow->chip_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + bias, + bias_stdev, + fringe_0, + fringe_1, + fringe_2, + sigma_ra, + sigma_dec, + ap_resid, + ap_resid_stdev, + zp_mean, + zp_stdev, + fwhm_major, + fwhm_minor, + dtime_detrend, + dtime_photom, + dtime_astrom, + hostname, + n_stars, + n_extended, + n_cr, + n_astrom, + path_base, + code ); @@ -649,7 +648,7 @@ } - // should we stop here or proceed on to the warp stage? // NULL for end_stage means go as far as possible - // we can't start a warp run unlesss tess_id is defined + // XXX true for fake too? + // XXX we can't start a warp run unlesss tess_id is defined if ((pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam")) || (pendingRow->tess_id == NULL) || psStrcasestr(pendingRow->tess_id, "NULL")) { @@ -666,8 +665,10 @@ // else continue on... - if (!pxwarpQueueByCamID(config, + if (!pxfakeQueueByCamID(config, pendingRow->cam_id, pendingRow->workdir, pendingRow->label, + pendingRow->reduction, + pendingRow->expgroup, pendingRow->dvodb, pendingRow->tess_id, @@ -678,5 +679,5 @@ psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PS_ERR_UNKNOWN, false, "failed to queue new warpRun"); + psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); psFree(pendingRow); return false; Index: /branches/faketool/ippTools/src/faketool.c =================================================================== --- /branches/faketool/ippTools/src/faketool.c (revision 17937) +++ /branches/faketool/ippTools/src/faketool.c (revision 17938) @@ -32,5 +32,4 @@ #include "faketool.h" -#include "camtool.h" static bool queueMode(pxConfig *config); @@ -171,5 +170,5 @@ // find the exp_id of all the exposures that we want to queue up. - psString query = pxDataGet("faketool_find_rawexp.sql"); + psString query = pxDataGet("faketool_find_camrun.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -179,5 +178,5 @@ if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psString whereClause = psDBGenerateWhereConditionSQL(where, "camRun"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -219,12 +218,12 @@ } - // loop over our list of exp_ids + // loop over our list of cam_ids for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; bool status; - psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + psS64 cam_id = psMetadataLookupS64(&status, md, "cam_id"); if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for cam_id"); psFree(output); return false; @@ -232,10 +231,10 @@ // queue the exp - if (!pxfakeQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!pxfakeQueueByCamID(config, cam_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); } psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue exp_id: %" PRId64, exp_id); + "failed to trying to queue cam_id: %" PRId64, cam_id); psFree(output); return false; @@ -448,38 +447,17 @@ // fake_id, ext_tag, class_id are required - PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", true, false); - PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); - PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", true, false); + PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // optional - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); - PXOPT_LOOKUP_F32(bg, config->args, "-bg", false, false); - PXOPT_LOOKUP_F32(bg_stdev, config->args, "-bg_stdev", false, false); - PXOPT_LOOKUP_F32(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); - PXOPT_LOOKUP_F32(bias, config->args, "-bias", false, false); - PXOPT_LOOKUP_F32(bias_stdev, config->args, "-bias_stdev", false, false); - PXOPT_LOOKUP_F32(fringe_0, config->args, "-fringe_0", false, false); - PXOPT_LOOKUP_F32(fringe_1, config->args, "-fringe_1", false, false); - PXOPT_LOOKUP_F32(fringe_2, config->args, "-fringe_2", false, false); - PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); - PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); - PXOPT_LOOKUP_F32(ap_resid, config->args, "-ap_resid", false, false); - PXOPT_LOOKUP_F32(ap_resid_stdev, config->args, "-ap_resid_stdev", false, false); - PXOPT_LOOKUP_F32(zp_mean, config->args, "-zp_mean", false, false); - PXOPT_LOOKUP_F32(zp_stdev, config->args, "-zp_stdev", false, false); - PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); - PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); - PXOPT_LOOKUP_F32(dtime_detrend, config->args, "-dtime_detrend", false, false); - PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); - PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); - PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); - PXOPT_LOOKUP_F32(n_stars, config->args, "-n_stars", false, false); - PXOPT_LOOKUP_F32(n_extended, config->args, "-n_extended", false, false); - PXOPT_LOOKUP_F32(n_cr, config->args, "-n_cr", false, false); - PXOPT_LOOKUP_F32(n_astrom, config->args, "-n_astrom", false, false); - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + + PXOPT_LOOKUP_F32(dtime_fake, config->args, "-dtime_fake", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); // default values - PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); if (!psDBTransaction(config->dbh)) { @@ -493,30 +471,9 @@ class_id, uri, - bg, - bg_stdev, - bg_mean_stdev, - bias, - bias_stdev, - fringe_0, - fringe_1, - fringe_2, - sigma_ra, - sigma_dec, - ap_resid, - ap_resid_stdev, - zp_mean, - zp_stdev, - fwhm_major, - fwhm_minor, - dtime_detrend, - dtime_photom, - dtime_astrom, + dtime_fake, hostname, - n_stars, - n_extended, - n_cr, - n_astrom, path_base, - code + code, + NULL // epoch )) { // rollback @@ -984,10 +941,8 @@ // camQueueFakeID() can only be run after fakeRun.state has been set to // stop - if (!pxcamQueueByFakeID(config, + if (!pxwarpQueueByFakeID(config, fakeRun->fake_id, fakeRun->workdir, fakeRun->label, - fakeRun->reduction, - fakeRun->expgroup, fakeRun->dvodb, fakeRun->tess_id, Index: /branches/faketool/ippTools/src/pxtools.h =================================================================== --- /branches/faketool/ippTools/src/pxtools.h (revision 17937) +++ /branches/faketool/ippTools/src/pxtools.h (revision 17938) @@ -28,13 +28,16 @@ #include +#include "pxconfig.h" #include "pxtoolsErrorCodes.h" + +#include "pxcam.h" +#include "pxchip.h" +#include "pxdata.h" +#include "pxfake.h" +#include "pxwarp.h" #include "pxio.h" -#include "pxdata.h" +#include "pxregister.h" #include "pxtag.h" -#include "pxregister.h" -#include "pxchip.h" -#include "pxcam.h" #include "pxtree.h" -#include "pxconfig.h" # define MAX_ROWS 10e9 Index: /branches/faketool/ippTools/src/pxwarp.c =================================================================== --- /branches/faketool/ippTools/src/pxwarp.c (revision 17937) +++ /branches/faketool/ippTools/src/pxwarp.c (revision 17938) @@ -135,6 +135,6 @@ -bool pxwarpQueueByCamID(pxConfig *config, - psS64 cam_id, +bool pxwarpQueueByFakeID(pxConfig *config, + psS64 fake_id, char *workdir, char *label, @@ -148,5 +148,5 @@ if (!warpRunInsert(config->dbh, 0, // ID - cam_id, + fake_id, "warp", // mode "run", // state Index: /branches/faketool/ippTools/src/pxwarp.h =================================================================== --- /branches/faketool/ippTools/src/pxwarp.h (revision 17937) +++ /branches/faketool/ippTools/src/pxwarp.h (revision 17938) @@ -30,6 +30,6 @@ bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); -bool pxwarpQueueByCamID(pxConfig *config, - psS64 cam_id, +bool pxwarpQueueByFakeID(pxConfig *config, + psS64 fake_id, char *workdir, char *label, Index: /branches/faketool/ippTools/src/warptool.c =================================================================== --- /branches/faketool/ippTools/src/warptool.c (revision 17937) +++ /branches/faketool/ippTools/src/warptool.c (revision 17938) @@ -111,5 +111,5 @@ // required options - PXOPT_LOOKUP_STR(cam_id, config->args, "-cam_id", true, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", true, false); PXOPT_LOOKUP_STR(mode, config->args, "-mode", true, false); // check mode @@ -133,5 +133,5 @@ warpRunRow *warpRun = warpRunRowAlloc( 0, // ID - (psS64)atoll(cam_id), + (psS64)atoll(fake_id), mode, "run", // state @@ -176,5 +176,5 @@ PXOPT_LOOKUP_STR(warp_id, config->args, "-warp_id", false, false); - PXOPT_LOOKUP_STR(cam_id, config->args, "-cam_id", false, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", false, false); PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); @@ -190,8 +190,8 @@ } - // convert cam_id into a psS64 - if (cam_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "cam_id", 0, "==", (psS64)atoll(cam_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item cam_id"); + // convert fake_id into a psS64 + if (fake_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "fake_id", 0, "==", (psS64)atoll(fake_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fake_id"); psFree(where); return false; @@ -543,5 +543,11 @@ } } else { - warpSkyCellMapInsert(config->dbh, (psS64)atoll(warp_id), NULL, NULL, 0, NULL, code); + warpSkyCellMapInsert(config->dbh, + (psS64)atoll(warp_id), + NULL, // skycell_id + NULL, // tess_id + NULL, // class_id + code // fault + ); } @@ -1028,5 +1034,5 @@ PXOPT_LOOKUP_STR(warp_id, config->args, "-warp_id", false, false); - PXOPT_LOOKUP_STR(cam_id, config->args, "-cam_id", false, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", false, false); PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); @@ -1045,8 +1051,8 @@ PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); - // convert cam_id into a psS64 - if (cam_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "camRun.cam_id", 0, "==", (psS64)atoll(cam_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item cam_id"); + // convert fake_id into a psS64 + if (fake_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "fakeRun.fake_id", 0, "==", (psS64)atoll(fake_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fake_id"); psFree(where); return false; Index: /branches/faketool/ippTools/src/warptoolConfig.c =================================================================== --- /branches/faketool/ippTools/src/warptoolConfig.c (revision 17937) +++ /branches/faketool/ippTools/src/warptoolConfig.c (revision 17938) @@ -49,5 +49,5 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-fake_id", 0, "define camtool ID (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-mode", 0, @@ -72,10 +72,10 @@ // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - // XXX need to allow multiple cam_ids + // XXX need to allow multiple fake_ids psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); - // XXX need to allow multiple cam_ids - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, - "search by cam_id", NULL); + // XXX need to allow multiple fake_ids + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by fake_id", NULL); // XXX need to allow multiple chip_ids psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, @@ -176,5 +176,5 @@ psMetadataAddStr(expArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); - psMetadataAddStr(expArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(expArgs, PS_LIST_TAIL, "-fake_id", 0, "search by camtool ID", NULL); psMetadataAddU64(expArgs, PS_LIST_TAIL, "-limit", 0, @@ -187,5 +187,5 @@ psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); - psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by camtool ID", NULL); psMetadataAddU64(imfileArgs, PS_LIST_TAIL, "-limit", 0, @@ -279,5 +279,5 @@ psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure tag", NULL); - psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "define phase 3 version of exposure tag", NULL); psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, @@ -288,5 +288,5 @@ // -revertwarped psMetadata *revertwarpedArgs = psMetadataAlloc(); - // XXX need to allow multiple cam_ids + // XXX need to allow multiple fake_ids psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); @@ -295,7 +295,7 @@ psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, "searcy by tessellation ID", NULL); - // XXX need to allow multiple cam_ids - psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-cam_id", 0, - "search by cam_id", NULL); + // XXX need to allow multiple fake_ids + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by fake_id", NULL); // XXX need to allow multiple chip_ids psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-chip_id", 0, Index: /trunk/ippTools/scripts/faketest.sh =================================================================== --- /trunk/ippTools/scripts/faketest.sh (revision 17938) +++ /trunk/ippTools/scripts/faketest.sh (revision 17938) @@ -0,0 +1,19 @@ +#!//bin/sh + +set -o verbose + +camtest.sh || exit 1 + +#faketool -pendingexp || exit 1 +faketool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + faketool -addprocessedimfile -fake_id 1 -exp_id 1 -class_id $ID -uri file://chipp-t10.$ID -path_base file:///foo || exit 1 +done; + +for ID in `seq 0 3`; do + faketool -addprocessedimfile -fake_id 2 -exp_id 2 -class_id $ID -uri file://chipp-t11.$ID -path_base file:///foo || exit 1 +done; + +faketool -pendingimfile || exit 1 +faketool -processedimfile || exit 1 Index: /trunk/ippTools/share/faketool_completely_processed_exp.sql =================================================================== --- /trunk/ippTools/share/faketool_completely_processed_exp.sql (revision 17938) +++ /trunk/ippTools/share/faketool_completely_processed_exp.sql (revision 17938) @@ -0,0 +1,40 @@ +-- the output of this query must match the format of fakeRun row +SELECT DISTINCT + fake_id, + cam_id, + state, + workdir, + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage, + epoch +FROM + (SELECT + fakeRun.*, + rawImfile.class_id as rawimfile_class_id, + fakeProcessedImfile.class_id + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + JOIN rawImfile + USING(exp_id) + LEFT JOIN fakeProcessedImfile + ON fakeRun.fake_id = fakeProcessedImfile.fake_id + AND rawImfile.exp_id = fakeProcessedImfile.exp_id + AND rawImfile.class_id = fakeProcessedImfile.class_id + WHERE + fakeRun.state = 'run' + GROUP BY + fakeRun.fake_id, + rawExp.exp_id + HAVING + COUNT(rawImfile.class_id) = COUNT(fakeProcessedImfile.class_id) + AND SUM(fakeProcessedImfile.fault) = 0 + ) as Foo Index: /trunk/ippTools/share/faketool_pendingimfile.sql =================================================================== --- /trunk/ippTools/share/faketool_pendingimfile.sql (revision 17938) +++ /trunk/ippTools/share/faketool_pendingimfile.sql (revision 17938) @@ -0,0 +1,45 @@ +SELECT DISTINCT * FROM + -- the subselect is so where criteria can be specified without knowing + -- which table the field came from + (SELECT + fakeRun.*, + chipProcessedImfile.exp_id, + chipProcessedImfile.class_id, + chipProcessedImfile.uri, + chipProcessedImfile.bg, + chipProcessedImfile.bg_stdev, + chipProcessedImfile.bg_mean_stdev, + chipProcessedImfile.fringe_0, + chipProcessedImfile.fringe_1, + chipProcessedImfile.fringe_2, + chipProcessedImfile.ap_resid, + chipProcessedImfile.ap_resid_stdev, + chipProcessedImfile.sigma_ra, + chipProcessedImfile.sigma_dec, + chipProcessedImfile.n_stars, + chipProcessedImfile.n_extended, + chipProcessedImfile.n_cr, + chipProcessedImfile.n_astrom, + chipProcessedImfile.path_base, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipProcessedImfile.exp_id = rawExp.exp_id + LEFT JOIN fakeProcessedImfile + ON fakeRun.fake_id = fakeProcessedImfile.fake_id + LEFT JOIN fakeMask + ON fakeRun.label = fakeMask.label + WHERE + fakeProcessedImfile.fake_id IS NULL + AND fakeProcessedImfile.class_id IS NULL + AND fakeMask.label IS NULL + ) as foo Index: /trunk/ippTools/share/faketool_processedimfile.sql =================================================================== --- /trunk/ippTools/share/faketool_processedimfile.sql (revision 17938) +++ /trunk/ippTools/share/faketool_processedimfile.sql (revision 17938) @@ -0,0 +1,17 @@ +SELECT DISTINCT + fakeProcessedImfile.*, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM fakeRun +JOIN fakeProcessedImfile + USING(fake_id) +JOIN rawExp + ON fakeProcessedImfile.exp_id = rawExp.exp_id +WHERE +-- bogus test; just here so there there is a 'WHERE' stmt to append +-- conditionals too + fakeProcessedImfile.exp_id is NOT NULL + Index: /trunk/ippTools/share/faketool_queue_cam_id.sql =================================================================== --- /trunk/ippTools/share/faketool_queue_cam_id.sql (revision 17938) +++ /trunk/ippTools/share/faketool_queue_cam_id.sql (revision 17938) @@ -0,0 +1,17 @@ +INSERT INTO fakeRun + SElECT + 0, -- fake_id + cam_id, -- cam_id + '%s', -- state + '%s', -- workdir + '%s', -- label + '%s', -- reduction + '%s', -- expgroup + '%s', -- dvodb + '%s', -- tess_id + '%s', -- end_stage + NULL -- epoch + FROM camRun + WHERE + camRun.state = 'stop' + AND camRun.cam_id = %lld Index: /trunk/ippTools/src/faketool.h =================================================================== --- /trunk/ippTools/src/faketool.h (revision 17938) +++ /trunk/ippTools/src/faketool.h (revision 17938) @@ -0,0 +1,45 @@ +/* + * faketool.h + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef FAKETOOL_H +#define FAKETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + FAKETOOL_MODE_NONE = 0x0, + FAKETOOL_MODE_QUEUE, + FAKETOOL_MODE_UPDATERUN, + FAKETOOL_MODE_PENDINGIMFILE, + FAKETOOL_MODE_ADDPROCESSEDIMFILE, + FAKETOOL_MODE_PROCESSEDIMFILE, + FAKETOOL_MODE_REVERTPROCESSEDIMFILE, + FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, + FAKETOOL_MODE_BLOCK, + FAKETOOL_MODE_MASKED, + FAKETOOL_MODE_UNMASKED, + FAKETOOL_MODE_UNBLOCK, + FAKETOOL_MODE_RETRYPROCESSEDIMFILE, + FAKETOOL_MODE_PENDINGCLEANUP, + FAKETOOL_MODE_DONECLEANUP, +} FAKETOOLMode; + +pxConfig *faketoolConfig(pxConfig *config, int argc, char **argv); + +#endif // FAKETOOL_H Index: /trunk/ippTools/src/pxfake.c =================================================================== --- /trunk/ippTools/src/pxfake.c (revision 17938) +++ /trunk/ippTools/src/pxfake.c (revision 17938) @@ -0,0 +1,188 @@ +/* + * pxfake.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxcam.h" + +bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid fakeRun state: %s", state); + return false; + } + + char *query = "UPDATE fakeRun SET state = '%s' WHERE fake_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, fake_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for fake_id %" PRId64, fake_id); + return false; + } + + return true; +} + + +bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid fakeRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE fakeRun SET fakeRun.label = '%s' WHERE fake_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, label, fake_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for fake_id %" PRId64, fake_id); + return false; + } + + return true; +} + + +bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET fakeRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +psS64 pxfakeQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *reduction, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psTrace("pxtool", PS_LOG_INFO, "attemping to queue cam_id: %"PRId64, cam_id); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("faketool_queue_cam_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQuery(config->dbh, query, + "run", // state + workdir ? workdir : "NULL", + label ? label : "NULL", + reduction? reduction: "NULL", + expgroup ? expgroup : "NULL", + dvodb ? dvodb : "NULL", + tess_id ? tess_id : "NULL", + end_stage ? end_stage : "NULL", + (long long)cam_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have changed just one row"); + return false; + } + + return psDBLastInsertID(config->dbh); +} Index: /trunk/ippTools/src/pxfake.h =================================================================== --- /trunk/ippTools/src/pxfake.h (revision 17938) +++ /trunk/ippTools/src/pxfake.h (revision 17938) @@ -0,0 +1,44 @@ +/* + * pxfake.h + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXFAKE_H +#define PXFAKE_H 1 + +#include + +#include "pxtools.h" + +bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state); +bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label); +bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + + +psS64 pxfakeQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *reduction, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage); + + +#endif // PXFAKE_H