Index: /branches/cleanup/ippTools/configure.ac =================================================================== --- /branches/cleanup/ippTools/configure.ac (revision 18075) +++ /branches/cleanup/ippTools/configure.ac (revision 18076) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.36]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.38]) PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" Index: /branches/cleanup/ippTools/scripts/faketest.sh =================================================================== --- /branches/cleanup/ippTools/scripts/faketest.sh (revision 18076) +++ /branches/cleanup/ippTools/scripts/faketest.sh (revision 18076) @@ -0,0 +1,19 @@ +#!//bin/sh + +set -o verbose + +camtest.sh || exit 1 + +#faketool -pendingexp || exit 1 +faketool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + faketool -addprocessedimfile -fake_id 1 -exp_id 1 -class_id $ID -uri file://chipp-t10.$ID -path_base file:///foo || exit 1 +done; + +for ID in `seq 0 3`; do + faketool -addprocessedimfile -fake_id 2 -exp_id 2 -class_id $ID -uri file://chipp-t11.$ID -path_base file:///foo || exit 1 +done; + +faketool -pendingimfile || exit 1 +faketool -processedimfile || exit 1 Index: /branches/cleanup/ippTools/scripts/regtest.sh =================================================================== --- /branches/cleanup/ippTools/scripts/regtest.sh (revision 18075) +++ /branches/cleanup/ippTools/scripts/regtest.sh (revision 18076) @@ -33,5 +33,5 @@ $regtool -pendingexp|| exit 1 -$regtool -addprocessedexp -exp_id $exp_id1 -exp_name t10 -inst gpc -telescope ps1 -exp_tag batman.t10 -filelevel OTA -filter r -airmass 10 -ra 1 -decl 2 -exp_type object -exp_time 0 -bg 10 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 45 -posang 10 -object dog -dateobs "2006-10-20T10:10:10Z" -label foobar $* || exit 1 +$regtool -addprocessedexp -exp_id $exp_id1 -exp_name t10 -inst gpc -telescope ps1 -exp_tag batman.t10 -filelevel OTA -filter r -airmass 10 -ra 1 -decl 2 -exp_type object -exp_time 0 -bg 10 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 45 -posang 10 -object dog -dateobs "2006-10-20T10:10:10Z" -label foobar -tess_id moldyshoe $* || exit 1 $regtool -pendingimfile || exit 1 @@ -43,3 +43,3 @@ $regtool -pendingexp|| exit 1 -$regtool -addprocessedexp -exp_id $exp_id2 -exp_name t11 -inst gpc -telescope ps1 -exp_tag batman.t11 -filelevel OTA -filter r -airmass 11 -ra 1 -decl 2 -exp_type object -exp_time 0 -bg 11 -bg_stdev 1 -bg_mean_stdev 11 -alt 11 -az 11 -ccd_temp 45 -posang 11 -object dog -dateobs 2006-10-20T10:10:10Z -label foobar $* || exit 1 +$regtool -addprocessedexp -exp_id $exp_id2 -exp_name t11 -inst gpc -telescope ps1 -exp_tag batman.t11 -filelevel OTA -filter r -airmass 11 -ra 1 -decl 2 -exp_type object -exp_time 0 -bg 11 -bg_stdev 1 -bg_mean_stdev 11 -alt 11 -az 11 -ccd_temp 45 -posang 11 -object dog -dateobs 2006-10-20T10:10:10Z -label foobar -tess_id moldyshoe $* || exit 1 Index: /branches/cleanup/ippTools/scripts/warptest.sh =================================================================== --- /branches/cleanup/ippTools/scripts/warptest.sh (revision 18075) +++ /branches/cleanup/ippTools/scripts/warptest.sh (revision 18076) @@ -3,7 +3,8 @@ set -o verbose -camtest.sh || exit 1 +faketest.sh || exit 1 -#warptool -definerun -mode warp -workdir file:///foo || exit 1 +#warp tool -definerun -fake_id 1 -mode warp -workdir file:///foo + #warptool -addinputexp -warp_id 1 -cam_id 1 || exit 1 #warptool -addinputexp -warp_id 1 -cam_id 2 || exit 1 @@ -15,6 +16,4 @@ warptool -tooverlap -warp_id 1 || exit 1 - -exit 1 warptool -addoverlap -mapfile mapfile.txt || exit 1 Index: /branches/cleanup/ippTools/share/Makefile.am =================================================================== --- /branches/cleanup/ippTools/share/Makefile.am (revision 18075) +++ /branches/cleanup/ippTools/share/Makefile.am (revision 18076) @@ -1,7 +1,7 @@ dist_pkgdata_DATA = \ + camtool_find_chip_id.sql \ camtool_find_pendingexp.sql \ camtool_find_pendingimfile.sql \ camtool_find_processedexp.sql \ - camtool_find_chip_id.sql \ camtool_queue_chip_id.sql \ camtool_reset_faulted_runs.sql \ @@ -19,11 +19,7 @@ detselect_search.sql \ detselect_select.sql \ - dettool_tocorrectimfile.sql \ - dettool_tocorrectexp.sql \ - dettool_stop_completed_correct_runs.sql \ dettool_find_completed_runs.sql \ dettool_normalizedstat.sql \ dettool_raw.sql \ - dettool_runs.sql \ dettool_revertdetrunsummary.sql \ dettool_revertnormalizedexp.sql \ @@ -35,5 +31,9 @@ dettool_revertresidimfile.sql\ dettool_revertstacked.sql \ + dettool_runs.sql \ dettool_start_new_iteration.sql \ + dettool_stop_completed_correct_runs.sql \ + dettool_tocorrectexp.sql \ + dettool_tocorrectimfile.sql \ dettool_todetrunsummary.sql \ dettool_tonormalizedexp.sql \ @@ -45,24 +45,32 @@ dettool_tostacked.sql \ difftool_inputskyfile.sql \ + difftool_queue.sql \ + difftool_revertdiffskyfile.sql \ difftool_skyfile.sql \ difftool_todiffskyfile.sql \ - difftool_revertdiffskyfile.sql \ - difftool_queue.sql \ + faketool_completely_processed_exp.sql \ + faketool_find_camrun.sql \ + faketool_find_pendingexp.sql \ + faketool_pendingimfile.sql \ + faketool_processedimfile.sql \ + faketool_queue_cam_id.sql \ + faketool_revertprocessedimfile.sql \ + faketool_unmasked.sql \ + flatcorr_completely_processed_chiprun.sql \ flatcorr_create_tmp_chiprundone.sql \ - flatcorr_completely_processed_chiprun.sql \ flatcorr_find_completed_floatcorruns.sql \ flatcorr_find_processedimfiles.sql \ - magictool_toprocess_inputs.sql \ - magictool_toprocess_tree.sql \ - magictool_tomask.sql \ - magictool_toskyfilemask.sql \ magictool_find_complete_diffed_exposures.sql \ magictool_find_complete_warpruns.sql \ magictool_inputskyfile.sql \ + magictool_tomask.sql \ + magictool_toprocess_inputs.sql \ + magictool_toprocess_tree.sql \ + magictool_toskyfilemask.sql \ + pxadmin_create_tables.sql \ + pxadmin_drop_tables.sql \ pztool_find_completed_exp.sql \ pztool_pendingimfile.sql \ pztool_revertcopied.sql \ - pxadmin_create_tables.sql \ - pxadmin_drop_tables.sql \ regtool_pendingexp.sql \ regtool_pendingimfile.sql \ @@ -80,5 +88,5 @@ warptool_scmap.sql \ warptool_tooverlap.sql \ + warptool_revertwarped.sql \ warptool_towarped.sql \ - warptool_warped.sql \ - warptool_revertwarped.sql + warptool_warped.sql Index: /branches/cleanup/ippTools/share/camtool_find_pendingexp.sql =================================================================== --- /branches/cleanup/ippTools/share/camtool_find_pendingexp.sql (revision 18075) +++ /branches/cleanup/ippTools/share/camtool_find_pendingexp.sql (revision 18076) @@ -20,5 +20,5 @@ -- ON chipProcessedImfile.exp_id = rawExp.exp_id LEFT JOIN camProcessedExp - ON camRun.cam_id = camProcessedExp.cam_id + USING(cam_id) LEFT JOIN camMask ON camRun.label = camMask.label Index: /branches/cleanup/ippTools/share/camtool_find_pendingimfile.sql =================================================================== --- /branches/cleanup/ippTools/share/camtool_find_pendingimfile.sql (revision 18075) +++ /branches/cleanup/ippTools/share/camtool_find_pendingimfile.sql (revision 18076) @@ -34,5 +34,5 @@ ON chipProcessedImfile.exp_id = rawExp.exp_id LEFT JOIN camProcessedExp - ON camRun.cam_id = camProcessedExp.cam_id + USING(cam_id) LEFT JOIN camMask ON camRun.label = camMask.label Index: /branches/cleanup/ippTools/share/camtool_find_processedexp.sql =================================================================== --- /branches/cleanup/ippTools/share/camtool_find_processedexp.sql (revision 18075) +++ /branches/cleanup/ippTools/share/camtool_find_processedexp.sql (revision 18076) @@ -6,5 +6,7 @@ rawExp.telescope, rawExp.filelevel -FROM camProcessedExp +FROM camRun +JOIN camProcessedExp + USING(cam_id) JOIN chipRun USING(chip_id) Index: /branches/cleanup/ippTools/share/difftool_inputskyfile.sql =================================================================== --- /branches/cleanup/ippTools/share/difftool_inputskyfile.sql (revision 18075) +++ /branches/cleanup/ippTools/share/difftool_inputskyfile.sql (revision 18076) @@ -19,14 +19,20 @@ JOIN warpRun ON diffInputSkyfile.warp_id = warpRun.warp_id - JOIN camProcessedExp - ON warpRun.cam_id = camProcessedExp.cam_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + USING(chip_id) JOIN chipProcessedImfile - ON chipRun.chip_id = chipProcessedImfile.chip_id + USING(chip_id) JOIN rawExp - ON chipProcessedImfile.exp_id = rawExp.exp_id + ON chipRun.exp_id = rawExp.exp_id WHERE diffRun.state = 'run' + AND warpRun.state = 'stop' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' UNION SELECT @@ -49,13 +55,19 @@ JOIN warpRun ON stackInputSkyfile.warp_id = warpRun.warp_id - JOIN camProcessedExp - ON warpRun.cam_id = camProcessedExp.cam_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + USING(chip_id) JOIN chipProcessedImfile - ON chipRun.chip_id = chipProcessedImfile.chip_id + USING(chip_id) JOIN rawExp - ON chipProcessedImfile.exp_id = rawExp.exp_id + ON chipRun.exp_id = rawExp.exp_id WHERE diffRun.state = 'run' + AND warpRun.state = 'stop' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' ) as Foo Index: /branches/cleanup/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- /branches/cleanup/ippTools/share/difftool_todiffskyfile.sql (revision 18075) +++ /branches/cleanup/ippTools/share/difftool_todiffskyfile.sql (revision 18076) @@ -14,4 +14,6 @@ JOIN warpRun ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) JOIN camRun USING(cam_id) @@ -24,3 +26,7 @@ WHERE diffRun.state = 'run' + AND warpRun.state = 'stop' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' AND diffSkyfile.diff_id IS NULL Index: /branches/cleanup/ippTools/share/faketool_completely_processed_exp.sql =================================================================== --- /branches/cleanup/ippTools/share/faketool_completely_processed_exp.sql (revision 18076) +++ /branches/cleanup/ippTools/share/faketool_completely_processed_exp.sql (revision 18076) @@ -0,0 +1,40 @@ +-- the output of this query must match the format of fakeRun row +SELECT DISTINCT + fake_id, + cam_id, + state, + workdir, + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage, + epoch +FROM + (SELECT + fakeRun.*, + rawImfile.class_id as rawimfile_class_id, + fakeProcessedImfile.class_id + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + JOIN rawImfile + USING(exp_id) + LEFT JOIN fakeProcessedImfile + ON fakeRun.fake_id = fakeProcessedImfile.fake_id + AND rawImfile.exp_id = fakeProcessedImfile.exp_id + AND rawImfile.class_id = fakeProcessedImfile.class_id + WHERE + fakeRun.state = 'run' + GROUP BY + fakeRun.fake_id, + rawExp.exp_id + HAVING + COUNT(rawImfile.class_id) = COUNT(fakeProcessedImfile.class_id) + AND SUM(fakeProcessedImfile.fault) = 0 + ) as Foo Index: /branches/cleanup/ippTools/share/faketool_find_camrun.sql =================================================================== --- /branches/cleanup/ippTools/share/faketool_find_camrun.sql (revision 18076) +++ /branches/cleanup/ippTools/share/faketool_find_camrun.sql (revision 18076) @@ -0,0 +1,36 @@ +SELECT + * +FROM + (SELECT DISTINCT + camRun.cam_id, + chipRun.*, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.exp_tag, + rawExp.exp_type, + rawExp.filelevel, + rawExp.filter, + rawExp.airmass, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.sat_pixel_frac, + rawExp.bg, + rawExp.bg_stdev, + rawExp.bg_mean_stdev, + rawExp.alt, + rawExp.az, + rawExp.ccd_temp, + rawExp.posang, + rawExp.object, + rawExp.solang + FROM camRun + JOIN chipRun + using(chip_id) + JOIN rawExp + using(exp_id) + WHERE + camRun.state = 'stop' + AND chipRun.state = 'stop' +) as Foo Index: /branches/cleanup/ippTools/share/faketool_find_pendingexp.sql =================================================================== --- /branches/cleanup/ippTools/share/faketool_find_pendingexp.sql (revision 18076) +++ /branches/cleanup/ippTools/share/faketool_find_pendingexp.sql (revision 18076) @@ -0,0 +1,34 @@ +-- this query is used by both camtool -pendingexp & camtool -addprocessedexp it +-- does a little more work then is necessary for -addprocessed but it seems +-- "cleaner" to use the same query for both cases +SELECT * FROM + (SELECT + fakeRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + LEFT JOIN fakeProcessedImfile + USING(fake_id) + LEFT JOIN fakeMask + ON fakeRun.label = fakeMask.label + WHERE + fakeRun.state = 'run' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' + AND fakeMask.label IS NULL + AND fakeProcessedImfile.fake_id IS NULL + GROUP BY + fakeRun.fake_id + ) as Foo Index: /branches/cleanup/ippTools/share/faketool_pendingimfile.sql =================================================================== --- /branches/cleanup/ippTools/share/faketool_pendingimfile.sql (revision 18076) +++ /branches/cleanup/ippTools/share/faketool_pendingimfile.sql (revision 18076) @@ -0,0 +1,48 @@ +SELECT DISTINCT * FROM + -- the subselect is so where criteria can be specified without knowing + -- which table the field came from + (SELECT + fakeRun.*, + chipProcessedImfile.exp_id, + chipProcessedImfile.class_id, + chipProcessedImfile.uri, + chipProcessedImfile.bg, + chipProcessedImfile.bg_stdev, + chipProcessedImfile.bg_mean_stdev, + chipProcessedImfile.fringe_0, + chipProcessedImfile.fringe_1, + chipProcessedImfile.fringe_2, + chipProcessedImfile.ap_resid, + chipProcessedImfile.ap_resid_stdev, + chipProcessedImfile.sigma_ra, + chipProcessedImfile.sigma_dec, + chipProcessedImfile.n_stars, + chipProcessedImfile.n_extended, + chipProcessedImfile.n_cr, + chipProcessedImfile.n_astrom, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN camProcessedExp + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipProcessedImfile.exp_id = rawExp.exp_id + LEFT JOIN fakeProcessedImfile + ON fakeRun.fake_id = fakeProcessedImfile.fake_id + LEFT JOIN fakeMask + ON fakeRun.label = fakeMask.label + WHERE + fakeProcessedImfile.fake_id IS NULL + AND fakeProcessedImfile.class_id IS NULL + AND fakeMask.label IS NULL + ) as foo Index: /branches/cleanup/ippTools/share/faketool_processedimfile.sql =================================================================== --- /branches/cleanup/ippTools/share/faketool_processedimfile.sql (revision 18076) +++ /branches/cleanup/ippTools/share/faketool_processedimfile.sql (revision 18076) @@ -0,0 +1,17 @@ +SELECT DISTINCT + fakeProcessedImfile.*, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM fakeRun +JOIN fakeProcessedImfile + USING(fake_id) +JOIN rawExp + ON fakeProcessedImfile.exp_id = rawExp.exp_id +WHERE +-- bogus test; just here so there there is a 'WHERE' stmt to append +-- conditionals too + fakeProcessedImfile.exp_id is NOT NULL + Index: /branches/cleanup/ippTools/share/faketool_queue_cam_id.sql =================================================================== --- /branches/cleanup/ippTools/share/faketool_queue_cam_id.sql (revision 18076) +++ /branches/cleanup/ippTools/share/faketool_queue_cam_id.sql (revision 18076) @@ -0,0 +1,17 @@ +INSERT INTO fakeRun + SElECT + 0, -- fake_id + cam_id, -- cam_id + '%s', -- state + '%s', -- workdir + '%s', -- label + '%s', -- reduction + '%s', -- expgroup + '%s', -- dvodb + '%s', -- tess_id + '%s', -- end_stage + NULL -- epoch + FROM camRun + WHERE + camRun.state = 'stop' + AND camRun.cam_id = %lld Index: /branches/cleanup/ippTools/share/faketool_revertprocessedimfile.sql =================================================================== --- /branches/cleanup/ippTools/share/faketool_revertprocessedimfile.sql (revision 18076) +++ /branches/cleanup/ippTools/share/faketool_revertprocessedimfile.sql (revision 18076) @@ -0,0 +1,8 @@ +DELETE FROM fakeProcessedImfile +USING fakeProcessedImfile, fakeRun, camRun, chipRun, rawExp +WHERE + fakeProcessedImfile.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND fakeProcessedImfile.fault != 0 Index: /branches/cleanup/ippTools/share/faketool_unmasked.sql =================================================================== --- /branches/cleanup/ippTools/share/faketool_unmasked.sql (revision 18076) +++ /branches/cleanup/ippTools/share/faketool_unmasked.sql (revision 18076) @@ -0,0 +1,11 @@ +SELECT + * +FROM + (SELECT + fakeRun.label, + count(fakeRun.fake_id) as n_fakeruns + FROM fakeRun + LEFT JOIN fakeMask + USING (label) + WHERE fakeMask.label IS NULL + GROUP BY fakeRun.label) as fakeUnmask Index: /branches/cleanup/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/cleanup/ippTools/share/pxadmin_create_tables.sql (revision 18075) +++ /branches/cleanup/ippTools/share/pxadmin_create_tables.sql (revision 18076) @@ -3,4 +3,5 @@ telescope VARCHAR(64), uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(camera, telescope) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -15,4 +16,5 @@ imfiles INT, fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(exp_name, camera, telescope), KEY(fault)) @@ -29,4 +31,5 @@ class_id VARCHAR(64), uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(exp_name, camera, telescope, class, class_id), KEY(file_id), @@ -40,4 +43,5 @@ telescope VARCHAR(64), state VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(exp_name, camera, telescope), KEY(state), @@ -54,4 +58,5 @@ uri VARCHAR(255), fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(exp_name, camera, telescope, class, class_id), KEY(fault), @@ -61,31 +66,651 @@ REFERENCES summitImfile(exp_name, camera, telescope, class, class_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE newExp (exp_id BIGINT AUTO_INCREMENT, tmp_exp_name VARCHAR(64), tmp_camera VARCHAR(64), tmp_telescope VARCHAR(64), state VARCHAR(64), workdir VARCHAR(255), workdir_state VARCHAR(64), reduction VARCHAR(64), dvodb VARCHAR(255), tess_id VARCHAR(64), end_stage VARCHAR(64), label VARCHAR(64), PRIMARY KEY(exp_id), KEY(exp_id), KEY(tmp_exp_name), KEY(tmp_camera), KEY(tmp_telescope), KEY(state), KEY(workdir_state), KEY(end_stage), KEY(label)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE newImfile (exp_id BIGINT, tmp_class_id VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_id, tmp_class_id), FOREIGN KEY (exp_id) REFERENCES newExp(exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE rawExp (exp_id BIGINT, exp_name VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), dateobs DATETIME, exp_tag VARCHAR(255), exp_type VARCHAR(64), filelevel VARCHAR(64), workdir VARCHAR(255), reduction VARCHAR(64), dvodb VARCHAR(255), tess_id VARCHAR(64), end_stage VARCHAR(64), filter VARCHAR(64), comment VARCHAR(80), airmass FLOAT, ra DOUBLE, decl DOUBLE, exp_time FLOAT, sat_pixel_frac FLOAT, bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, alt DOUBLE, az DOUBLE, ccd_temp FLOAT, posang DOUBLE, m1_x FLOAT, m1_y FLOAT, m1_z FLOAT, m1_tip FLOAT, m1_tilt FLOAT, m2_x FLOAT, m2_y FLOAT, m2_z FLOAT, m2_tip FLOAT, m2_tilt FLOAT, env_temperature FLOAT, env_humidity FLOAT, env_wind_speed FLOAT, env_wind_dir FLOAT, teltemp_m1 FLOAT, teltemp_m1cell FLOAT, teltemp_m2 FLOAT, teltemp_spider FLOAT, teltemp_truss FLOAT, teltemp_extra FLOAT, pon_time FLOAT, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, object VARCHAR(64), solang FLOAT, hostname VARCHAR(64), fault SMALLINT NOT NULL, PRIMARY KEY(exp_id), KEY(exp_name), KEY(end_stage), KEY(fault), FOREIGN KEY (exp_id) REFERENCES newExp(exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE rawImfile (exp_id BIGINT, exp_name VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), dateobs DATETIME, tmp_class_id VARCHAR(64), class_id VARCHAR(64), uri VARCHAR(255), exp_type VARCHAR(64), filelevel VARCHAR(64), filter VARCHAR(64), comment VARCHAR(80), airmass FLOAT, ra DOUBLE, decl DOUBLE, exp_time FLOAT, sat_pixel_frac FLOAT, bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, alt DOUBLE, az DOUBLE, ccd_temp FLOAT, posang DOUBLE, m1_x FLOAT, m1_y FLOAT, m1_z FLOAT, m1_tip FLOAT, m1_tilt FLOAT, m2_x FLOAT, m2_y FLOAT, m2_z FLOAT, m2_tip FLOAT, m2_tilt FLOAT, env_temperature FLOAT, env_humidity FLOAT, env_wind_speed FLOAT, env_wind_dir FLOAT, teltemp_m1 FLOAT, teltemp_m1cell FLOAT, teltemp_m2 FLOAT, teltemp_spider FLOAT, teltemp_truss FLOAT, teltemp_extra FLOAT, pon_time FLOAT, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, object VARCHAR(64), hostname VARCHAR(64), fault SMALLINT NOT NULL, PRIMARY KEY(exp_id, class_id), KEY(tmp_class_id), KEY(fault), UNIQUE KEY(exp_id, tmp_class_id), FOREIGN KEY (exp_id, tmp_class_id) REFERENCES newImfile(exp_id, tmp_class_id)) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE newExp ( + exp_id BIGINT AUTO_INCREMENT, + tmp_exp_name VARCHAR(64), + tmp_camera VARCHAR(64), + tmp_telescope VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + label VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id), + KEY(exp_id), + KEY(tmp_exp_name), + KEY(tmp_camera), + KEY(tmp_telescope), + KEY(state), + KEY(workdir_state), + KEY(end_stage), + KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE newImfile ( + exp_id BIGINT, + tmp_class_id VARCHAR(64), + uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id, tmp_class_id), + FOREIGN KEY (exp_id) + REFERENCES newExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rawExp ( + exp_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + exp_tag VARCHAR(255), + exp_type VARCHAR(64), + filelevel VARCHAR(64), + workdir VARCHAR(255), + reduction VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + filter VARCHAR(64), + comment VARCHAR(80), + airmass FLOAT, + ra DOUBLE, + decl DOUBLE, + exp_time FLOAT, + sat_pixel_frac FLOAT, + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + alt DOUBLE, + az DOUBLE, + ccd_temp FLOAT, + posang DOUBLE, + m1_x FLOAT, + m1_y FLOAT, + m1_z FLOAT, + m1_tip FLOAT, + m1_tilt FLOAT, + m2_x FLOAT, + m2_y FLOAT, + m2_z FLOAT, + m2_tip FLOAT, + m2_tilt FLOAT, + env_temperature FLOAT, + env_humidity FLOAT, + env_wind_speed FLOAT, + env_wind_dir FLOAT, + teltemp_m1 FLOAT, + teltemp_m1cell FLOAT, + teltemp_m2 FLOAT, + teltemp_spider FLOAT, + teltemp_truss FLOAT, + teltemp_extra FLOAT, + pon_time FLOAT, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + object VARCHAR(64), + solang FLOAT, + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id), + KEY(exp_name), + KEY(end_stage), + KEY(fault), + FOREIGN KEY (exp_id) + REFERENCES newExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rawImfile ( + exp_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + tmp_class_id VARCHAR(64), + class_id VARCHAR(64), + uri VARCHAR(255), + exp_type VARCHAR(64), + filelevel VARCHAR(64), + filter VARCHAR(64), + comment VARCHAR(80), + airmass FLOAT, + ra DOUBLE, + decl DOUBLE, + exp_time FLOAT, + sat_pixel_frac FLOAT, + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + alt DOUBLE, + az DOUBLE, + ccd_temp FLOAT, + posang DOUBLE, + m1_x FLOAT, + m1_y FLOAT, + m1_z FLOAT, + m1_tip FLOAT, + m1_tilt FLOAT, + m2_x FLOAT, + m2_y FLOAT, + m2_z FLOAT, + m2_tip FLOAT, + m2_tilt FLOAT, + env_temperature FLOAT, + env_humidity FLOAT, + env_wind_speed FLOAT, + env_wind_dir FLOAT, + teltemp_m1 FLOAT, + teltemp_m1cell FLOAT, + teltemp_m2 FLOAT, + teltemp_spider FLOAT, + teltemp_truss FLOAT, + teltemp_extra FLOAT, + pon_time FLOAT, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + object VARCHAR(64), + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id, class_id), + KEY(tmp_class_id), + KEY(fault), + UNIQUE KEY(exp_id, tmp_class_id), + FOREIGN KEY (exp_id, tmp_class_id) + REFERENCES newImfile(exp_id, tmp_class_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE guidePendingExp (guide_id BIGINT AUTO_INCREMENT, exp_id BIGINT, recipe VARCHAR(64), PRIMARY KEY(guide_id), KEY(guide_id), KEY(exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE chipRun (chip_id BIGINT AUTO_INCREMENT, exp_id BIGINT, state VARCHAR(64), workdir VARCHAR(255), workdir_state VARCHAR(64), label VARCHAR(64), reduction VARCHAR(64), expgroup VARCHAR(64), dvodb VARCHAR(255), tess_id VARCHAR(64), end_stage VARCHAR(64), PRIMARY KEY(chip_id), KEY(chip_id), KEY(exp_id), KEY(state), KEY(workdir_state), KEY(label), KEY(expgroup), KEY(end_stage), INDEX(chip_id, exp_id), FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE chipProcessedImfile (chip_id BIGINT, exp_id BIGINT, class_id VARCHAR(64), uri VARCHAR(255), bg FLOAT, bg_stdev FLOAT, bg_mean_stdev FLOAT, bias FLOAT, bias_stdev FLOAT, fringe_0 FLOAT, fringe_1 FLOAT, fringe_2 FLOAT, sigma_ra FLOAT, sigma_dec FLOAT, ap_resid FLOAT, ap_resid_stdev FLOAT, zp_mean FLOAT, zp_stdev FLOAT, fwhm_major FLOAT, fwhm_minor FLOAT, dtime_detrend FLOAT, dtime_photom FLOAT, dtime_astrom FLOAT, hostname VARCHAR(64), n_stars INT, n_extended INT, n_cr INT, n_astrom INT, path_base VARCHAR(255), fault SMALLINT NOT NULL, PRIMARY KEY(chip_id, exp_id, class_id), KEY(fault), FOREIGN KEY (chip_id, exp_id) REFERENCES chipRun(chip_id, exp_id), FOREIGN KEY (exp_id, class_id) REFERENCES rawImfile(exp_id, class_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE chipMask (label VARCHAR(64), PRIMARY KEY(label)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE camRun (cam_id BIGINT AUTO_INCREMENT, chip_id BIGINT, state VARCHAR(64), workdir VARCHAR(255), workdir_state VARCHAR(64), label VARCHAR(64), reduction VARCHAR(64), expgroup VARCHAR(64), dvodb VARCHAR(255), tess_id VARCHAR(64), end_stage VARCHAR(64), PRIMARY KEY(cam_id), KEY(cam_id), KEY(chip_id), KEY(state), KEY(workdir_state), KEY(label), KEY(expgroup), KEY(end_stage), INDEX(cam_id, chip_id), FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE camProcessedExp (cam_id BIGINT, chip_id BIGINT, uri VARCHAR(255), bg FLOAT, bg_stdev FLOAT, bg_mean_stdev FLOAT, bias FLOAT, bias_stdev FLOAT, fringe_0 FLOAT, fringe_1 FLOAT, fringe_2 FLOAT, sigma_ra FLOAT, sigma_dec FLOAT, ap_resid FLOAT, ap_resid_stdev FLOAT, zp_mean FLOAT, zp_stdev FLOAT, fwhm_major FLOAT, fwhm_minor FLOAT, dtime_detrend FLOAT, dtime_photom FLOAT, dtime_astrom FLOAT, hostname VARCHAR(64), n_stars INT, n_extended INT, n_cr INT, n_astrom INT, path_base VARCHAR(255), fault SMALLINT NOT NULL, PRIMARY KEY(cam_id, chip_id), KEY(fault), FOREIGN KEY (cam_id, chip_id) REFERENCES camRun(cam_id, chip_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE camMask (label VARCHAR(64), PRIMARY KEY(label)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE detRun (det_id BIGINT AUTO_INCREMENT, iteration INT, det_type VARCHAR(64), mode VARCHAR(64), state VARCHAR(64), filelevel VARCHAR(64), workdir VARCHAR(255), camera VARCHAR(64), telescope VARCHAR(64), exp_type VARCHAR(64), reduction VARCHAR(64), filter VARCHAR(64), airmass_min FLOAT, airmass_max FLOAT, exp_time_min FLOAT, exp_time_max FLOAT, ccd_temp_min FLOAT, ccd_temp_max FLOAT, posang_min DOUBLE, posang_max DOUBLE, registered DATETIME, time_begin DATETIME, time_end DATETIME, use_begin DATETIME, use_end DATETIME, solang_min FLOAT, solang_max FLOAT, label VARCHAR(64), parent INT, PRIMARY KEY(det_id), KEY(det_id), KEY(iteration), KEY(det_type), KEY(mode), KEY(state), KEY(label), KEY(parent), INDEX(det_id, iteration)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE detInputExp (det_id BIGINT, iteration INT, exp_id BIGINT, include TINYINT, PRIMARY KEY(det_id, iteration, exp_id), INDEX(det_id, exp_id), INDEX(det_id, iteration), FOREIGN KEY (det_id) REFERENCES detRun(det_id), FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE detProcessedImfile (det_id BIGINT, exp_id BIGINT, class_id VARCHAR(64), uri VARCHAR(255), recipe VARCHAR(64), bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, fringe_0 DOUBLE, fringe_1 DOUBLE, fringe_2 DOUBLE, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, path_base VARCHAR(255), fault SMALLINT NOT NULL, PRIMARY KEY(det_id, exp_id, class_id), KEY(fault), INDEX(det_id, class_id), INDEX(det_id, exp_id), FOREIGN KEY (det_id, exp_id) REFERENCES detInputExp(det_id, exp_id), FOREIGN KEY (exp_id, class_id) REFERENCES rawImfile(exp_id, class_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE detProcessedExp (det_id BIGINT, exp_id BIGINT, recipe VARCHAR(64), bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, fringe_0 DOUBLE, fringe_1 DOUBLE, fringe_2 DOUBLE, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, path_base VARCHAR(255), fault SMALLINT NOT NULL, PRIMARY KEY(det_id, exp_id), KEY(fault), FOREIGN KEY (det_id, exp_id) REFERENCES detInputExp(det_id, exp_id), FOREIGN KEY (det_id, exp_id) REFERENCES detProcessedImfile(det_id, exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE detStackedImfile (det_id BIGINT, iteration INT, class_id VARCHAR(64), uri VARCHAR(255), recipe VARCHAR(64), bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, fault SMALLINT NOT NULL, PRIMARY KEY(det_id, iteration, class_id), KEY(fault), FOREIGN KEY (det_id, iteration) REFERENCES detInputExp(det_id, iteration), FOREIGN KEY (det_id, class_id) REFERENCES detProcessedImfile(det_id, class_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE detNormalizedStatImfile (det_id BIGINT, iteration INT, class_id VARCHAR(64), norm FLOAT, fault SMALLINT NOT NULL, PRIMARY KEY(det_id, iteration, class_id), KEY(fault), FOREIGN KEY (det_id, iteration) REFERENCES detInputExp(det_id, iteration), FOREIGN KEY (det_id, iteration, class_id) REFERENCES detStackedImfile(det_id, iteration, class_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE detNormalizedImfile (det_id BIGINT, iteration INT, class_id VARCHAR(64), uri VARCHAR(255), bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, path_base VARCHAR(255), fault SMALLINT NOT NULL, PRIMARY KEY(det_id, iteration, class_id), KEY(fault), INDEX(det_id, iteration), FOREIGN KEY (det_id) REFERENCES detInputExp(det_id), FOREIGN KEY (det_id, iteration, class_id) REFERENCES detNormalizedStatImfile(det_id, iteration, class_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE detNormalizedExp (det_id BIGINT, iteration INT, recipe VARCHAR(64), bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, path_base VARCHAR(255), fault SMALLINT NOT NULL, PRIMARY KEY(det_id, iteration), KEY(fault), FOREIGN KEY (det_id, iteration) REFERENCES detInputExp(det_id, iteration), FOREIGN KEY (det_id, iteration) REFERENCES detNormalizedImfile(det_id, iteration)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE detResidImfile (det_id BIGINT, iteration INT, exp_id BIGINT, class_id VARCHAR(64), uri VARCHAR(255), recipe VARCHAR(64), bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, bg_skewness DOUBLE, bg_kurtosis DOUBLE, bin_stdev DOUBLE, fringe_0 DOUBLE, fringe_1 DOUBLE, fringe_2 DOUBLE, fringe_resid_0 DOUBLE, fringe_resid_1 DOUBLE, fringe_resid_2 DOUBLE, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, path_base VARCHAR(255), fault SMALLINT NOT NULL, PRIMARY KEY(det_id, iteration, exp_id, class_id), KEY(fault), INDEX(det_id, iteration, exp_id), FOREIGN KEY (det_id, iteration, exp_id) REFERENCES detInputExp(det_id, iteration, exp_id), FOREIGN KEY (det_id, exp_id, class_id) REFERENCES detProcessedImfile(det_id, exp_id, class_id), FOREIGN KEY (det_id, iteration) REFERENCES detNormalizedExp(det_id, iteration)) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipRun ( + chip_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + PRIMARY KEY(chip_id), + KEY(chip_id), KEY(exp_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(chip_id, exp_id), + FOREIGN KEY (exp_id) + REFERENCES rawExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipProcessedImfile ( + chip_id BIGINT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + bg FLOAT, + bg_stdev FLOAT, + bg_mean_stdev FLOAT, + bias FLOAT, + bias_stdev FLOAT, + fringe_0 FLOAT, + fringe_1 FLOAT, + fringe_2 FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + ap_resid FLOAT, + ap_resid_stdev FLOAT, + zp_mean FLOAT, + zp_stdev FLOAT, + fwhm_major FLOAT, + fwhm_minor FLOAT, + dtime_detrend FLOAT, + dtime_photom FLOAT, + dtime_astrom FLOAT, + hostname VARCHAR(64), + n_stars INT, + n_extended INT, + n_cr INT, + n_astrom INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(chip_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY (chip_id, exp_id) + REFERENCES chipRun(chip_id, exp_id), + FOREIGN KEY (exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camRun ( + cam_id BIGINT AUTO_INCREMENT, + chip_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + PRIMARY KEY(cam_id), + KEY(cam_id), + KEY(chip_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(cam_id, chip_id), + FOREIGN KEY (chip_id) + REFERENCES chipRun(chip_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camProcessedExp ( + cam_id BIGINT, + uri VARCHAR(255), + bg FLOAT, + bg_stdev FLOAT, + bg_mean_stdev FLOAT, + bias FLOAT, + bias_stdev FLOAT, + fringe_0 FLOAT, + fringe_1 FLOAT, + fringe_2 FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + ap_resid FLOAT, + ap_resid_stdev FLOAT, + zp_mean FLOAT, + zp_stdev FLOAT, + fwhm_major FLOAT, + fwhm_minor FLOAT, + dtime_detrend FLOAT, + dtime_photom FLOAT, + dtime_astrom FLOAT, + hostname VARCHAR(64), + n_stars INT, + n_extended INT, + n_cr INT, + n_astrom INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(cam_id), + KEY(fault), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeRun ( + fake_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id), + KEY(cam_id), + KEY(state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(fake_id, cam_id), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeProcessedImfile ( + fake_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + dtime_fake FLOAT, + hostname VARCHAR(64), + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id), + FOREIGN KEY (exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detRun ( + det_id BIGINT AUTO_INCREMENT, + iteration INT, + det_type VARCHAR(64), + mode VARCHAR(64), + state VARCHAR(64), + filelevel VARCHAR(64), + workdir VARCHAR(255), + camera VARCHAR(64), + telescope VARCHAR(64), + exp_type VARCHAR(64), + reduction VARCHAR(64), + filter VARCHAR(64), + airmass_min FLOAT, + airmass_max FLOAT, + exp_time_min FLOAT, + exp_time_max FLOAT, + ccd_temp_min FLOAT, + ccd_temp_max FLOAT, + posang_min DOUBLE, + posang_max DOUBLE, + registered DATETIME, + time_begin DATETIME, + time_end DATETIME, + use_begin DATETIME, + use_end DATETIME, + solang_min FLOAT, + solang_max FLOAT, + label VARCHAR(64), + parent INT, + PRIMARY KEY(det_id), + KEY(det_id), + KEY(iteration), + KEY(det_type), + KEY(mode), + KEY(state), + KEY(label), + KEY(parent), + INDEX(det_id, iteration)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detInputExp ( +det_id BIGINT, +iteration INT, +exp_id BIGINT, +include TINYINT, +PRIMARY KEY(det_id, iteration, exp_id), +INDEX(det_id, exp_id), +INDEX(det_id, iteration), +FOREIGN KEY (det_id) +REFERENCES detRun(det_id), +FOREIGN KEY (exp_id) +REFERENCES rawExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detProcessedImfile ( + det_id BIGINT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id, class_id), + KEY(fault), + INDEX(det_id, class_id), + INDEX(det_id, exp_id), + FOREIGN KEY (det_id, exp_id) + REFERENCES detInputExp(det_id, exp_id), + FOREIGN KEY (exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detProcessedExp ( + det_id BIGINT, + exp_id BIGINT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id), + KEY(fault), + FOREIGN KEY (det_id, exp_id) + REFERENCES detInputExp(det_id, exp_id), + FOREIGN KEY (det_id, exp_id) + REFERENCES detProcessedImfile(det_id, exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detStackedImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, class_id) + REFERENCES detProcessedImfile(det_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedStatImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + norm FLOAT, + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, iteration, class_id) + REFERENCES detStackedImfile(det_id, iteration, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + INDEX(det_id, iteration), + FOREIGN KEY (det_id) + REFERENCES detInputExp(det_id), + FOREIGN KEY (det_id, iteration, class_id) + REFERENCES detNormalizedStatImfile(det_id, iteration, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedExp ( + det_id BIGINT, + iteration INT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, iteration) + REFERENCES detNormalizedImfile(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detResidImfile ( + det_id BIGINT, + iteration INT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + bg_skewness DOUBLE, + bg_kurtosis DOUBLE, + bin_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + fringe_resid_0 DOUBLE, + fringe_resid_1 DOUBLE, + fringe_resid_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, exp_id, class_id), + KEY(fault), + INDEX(det_id, iteration, exp_id), + FOREIGN KEY (det_id, iteration, exp_id) + REFERENCES detInputExp(det_id, iteration, exp_id), + FOREIGN KEY (det_id, exp_id, class_id) + REFERENCES detProcessedImfile(det_id, exp_id, class_id), + FOREIGN KEY (det_id, iteration) + REFERENCES detNormalizedExp(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE detResidExp (det_id BIGINT, iteration INT, exp_id BIGINT, recipe VARCHAR(64), bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, bg_skewness DOUBLE, bg_kurtosis DOUBLE, bin_stdev DOUBLE, fringe_0 DOUBLE, fringe_1 DOUBLE, fringe_2 DOUBLE, fringe_resid_0 DOUBLE, fringe_resid_1 DOUBLE, fringe_resid_2 DOUBLE, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, path_base VARCHAR(255), accept TINYINT, fault SMALLINT NOT NULL, PRIMARY KEY(det_id, iteration, exp_id), KEY(fault), INDEX(det_id, iteration), FOREIGN KEY (det_id, iteration, exp_id) REFERENCES detInputExp(det_id, iteration, exp_id), FOREIGN KEY (det_id, iteration, exp_id) REFERENCES detResidImfile(det_id, iteration, exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE detRunSummary (det_id BIGINT, iteration INT, bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, accept TINYINT, fault SMALLINT NOT NULL, PRIMARY KEY(det_id, iteration), KEY(fault), FOREIGN KEY (det_id, iteration) REFERENCES detInputExp(det_id, iteration), FOREIGN KEY (det_id, iteration) REFERENCES detResidExp(det_id, iteration)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE detRegisteredImfile (det_id BIGINT, iteration INT, class_id VARCHAR(64), uri VARCHAR(255), bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, path_base VARCHAR(255), fault SMALLINT NOT NULL, PRIMARY KEY(det_id, iteration, class_id), KEY(fault), FOREIGN KEY (det_id, iteration) REFERENCES detRun(det_id, iteration)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpRun (warp_id BIGINT AUTO_INCREMENT, cam_id BIGINT, mode VARCHAR(64), state VARCHAR(64), workdir VARCHAR(255), workdir_state VARCHAR(64), label VARCHAR(64), dvodb VARCHAR(255), tess_id VARCHAR(64), end_stage VARCHAR(64), registered DATETIME, magiced TINYINT, PRIMARY KEY(warp_id), KEY(warp_id), KEY(cam_id), KEY(mode), KEY(state), KEY(workdir_state), KEY(label), KEY(end_stage), KEY(magiced), INDEX(warp_id, cam_id), FOREIGN KEY (cam_id) REFERENCES camProcessedExp(cam_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpSkyCellMap (warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64), cam_id BIGINT, class_id VARCHAR(64), fault SMALLINT, PRIMARY KEY(warp_id, skycell_id, tess_id, cam_id, class_id), KEY(fault), FOREIGN KEY (warp_id, cam_id) REFERENCES warpRun(warp_id, cam_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpSkyfile (warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64), uri VARCHAR(255), path_base VARCHAR(255), bg DOUBLE, bg_stdev DOUBLE, dtime_warp FLOAT, hostname VARCHAR(64), good_frac FLOAT, xmin INT, xmax INT, ymin INT, ymax INT, ignored TINYINT, fault SMALLINT, PRIMARY KEY(warp_id, skycell_id, tess_id), KEY(good_frac), KEY(ignored), KEY(fault), FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpMask (label VARCHAR(64), PRIMARY KEY(label)) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpRun ( + warp_id BIGINT AUTO_INCREMENT, + fake_id BIGINT, + mode VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + registered DATETIME, + magiced TINYINT, + PRIMARY KEY(warp_id), + KEY(warp_id), + KEY(fake_id), + KEY(mode), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(end_stage), + KEY(magiced), + INDEX(warp_id, fake_id), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyCellMap ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + class_id VARCHAR(64), + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id, class_id), + KEY(fault), + FOREIGN KEY (warp_id) + REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyfile ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + uri VARCHAR(255), + path_base VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_warp FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + xmin INT, + xmax INT, + ymin INT, + ymax INT, + ignored TINYINT, + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id), + KEY(good_frac), + KEY(ignored), KEY(fault), + FOREIGN KEY (warp_id, skycell_id, tess_id) + REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE stackRun (stack_id BIGINT AUTO_INCREMENT, state VARCHAR(64), workdir VARCHAR(255), dvodb VARCHAR(255), registered DATETIME, skycell_id VARCHAR(64), tess_id VARCHAR(64), PRIMARY KEY(stack_id), KEY(stack_id), KEY(state), KEY(skycell_id), KEY(tess_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE stackInputSkyfile (stack_id BIGINT, warp_id BIGINT, PRIMARY KEY(stack_id, warp_id), FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id), FOREIGN KEY (warp_id) REFERENCES warpSkyfile(warp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/cleanup/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /branches/cleanup/ippTools/share/pxadmin_drop_tables.sql (revision 18075) +++ /branches/cleanup/ippTools/share/pxadmin_drop_tables.sql (revision 18076) @@ -1,53 +1,56 @@ SET FOREIGN_KEY_CHECKS=0; -DROP TABLE pzDataStore; -DROP TABLE summitExp; -DROP TABLE summitImfile; -DROP TABLE pzDownloadExp; -DROP TABLE pzDownloadImfile; -DROP TABLE newExp; -DROP TABLE newImfile; -DROP TABLE rawExp; -DROP TABLE rawImfile; -DROP TABLE guidePendingExp; -DROP TABLE chipRun; -DROP TABLE chipProcessedImfile; -DROP TABLE chipMask; -DROP TABLE camRun; -DROP TABLE camProcessedExp; -DROP TABLE camMask; -DROP TABLE detRun; -DROP TABLE detInputExp; -DROP TABLE detProcessedImfile; -DROP TABLE detProcessedExp; -DROP TABLE detStackedImfile; -DROP TABLE detNormalizedStatImfile; -DROP TABLE detNormalizedImfile; -DROP TABLE detNormalizedExp; -DROP TABLE detResidImfile; -DROP TABLE detResidExp; -DROP TABLE detRunSummary; -DROP TABLE detRegisteredImfile; -DROP TABLE warpRun; -DROP TABLE warpSkyCellMap; -DROP TABLE warpSkyfile; -DROP TABLE warpMask; -DROP TABLE diffRun; -DROP TABLE diffInputSkyfile; -DROP TABLE diffSkyfile; -DROP TABLE stackRun; -DROP TABLE stackInputSkyfile; -DROP TABLE stackSumSkyfile; -DROP TABLE magicRun; -DROP TABLE magicInputSkyfile; -DROP TABLE magicTree; -DROP TABLE magicNodeResult; -DROP TABLE magicMask; -DROP TABLE magicSkyfileMask; -DROP TABLE calDB; -DROP TABLE calRun; -DROP TABLE flatcorrRun; -DROP TABLE flatcorrExp; -DROP TABLE pstampDataStore; -DROP TABLE pstampRequest; -DROP TABLE pstampJob; +DROP TABLE IF EXISTS pzDataStore; +DROP TABLE IF EXISTS summitExp; +DROP TABLE IF EXISTS summitImfile; +DROP TABLE IF EXISTS pzDownloadExp; +DROP TABLE IF EXISTS pzDownloadImfile; +DROP TABLE IF EXISTS newExp; +DROP TABLE IF EXISTS newImfile; +DROP TABLE IF EXISTS rawExp; +DROP TABLE IF EXISTS rawImfile; +DROP TABLE IF EXISTS guidePendingExp; +DROP TABLE IF EXISTS chipRun; +DROP TABLE IF EXISTS chipProcessedImfile; +DROP TABLE IF EXISTS chipMask; +DROP TABLE IF EXISTS camRun; +DROP TABLE IF EXISTS camProcessedExp; +DROP TABLE IF EXISTS camMask; +DROP TABLE IF EXISTS detRun; +DROP TABLE IF EXISTS detInputExp; +DROP TABLE IF EXISTS fakeRun; +DROP TABLE IF EXISTS fakeProcessedImfile; +DROP TABLE IF EXISTS fakeMask; +DROP TABLE IF EXISTS detProcessedImfile; +DROP TABLE IF EXISTS detProcessedExp; +DROP TABLE IF EXISTS detStackedImfile; +DROP TABLE IF EXISTS detNormalizedStatImfile; +DROP TABLE IF EXISTS detNormalizedImfile; +DROP TABLE IF EXISTS detNormalizedExp; +DROP TABLE IF EXISTS detResidImfile; +DROP TABLE IF EXISTS detResidExp; +DROP TABLE IF EXISTS detRunSummary; +DROP TABLE IF EXISTS detRegisteredImfile; +DROP TABLE IF EXISTS warpRun; +DROP TABLE IF EXISTS warpSkyCellMap; +DROP TABLE IF EXISTS warpSkyfile; +DROP TABLE IF EXISTS warpMask; +DROP TABLE IF EXISTS diffRun; +DROP TABLE IF EXISTS diffInputSkyfile; +DROP TABLE IF EXISTS diffSkyfile; +DROP TABLE IF EXISTS stackRun; +DROP TABLE IF EXISTS stackInputSkyfile; +DROP TABLE IF EXISTS stackSumSkyfile; +DROP TABLE IF EXISTS magicRun; +DROP TABLE IF EXISTS magicInputSkyfile; +DROP TABLE IF EXISTS magicTree; +DROP TABLE IF EXISTS magicNodeResult; +DROP TABLE IF EXISTS magicMask; +DROP TABLE IF EXISTS magicSkyfileMask; +DROP TABLE IF EXISTS calDB; +DROP TABLE IF EXISTS calRun; +DROP TABLE IF EXISTS flatcorrRun; +DROP TABLE IF EXISTS flatcorrExp; +DROP TABLE IF EXISTS pstampDataStore; +DROP TABLE IF EXISTS pstampRequest; +DROP TABLE IF EXISTS pstampJob; SET FOREIGN_KEY_CHECKS=1 Index: /branches/cleanup/ippTools/share/pztool_find_completed_exp.sql =================================================================== --- /branches/cleanup/ippTools/share/pztool_find_completed_exp.sql (revision 18075) +++ /branches/cleanup/ippTools/share/pztool_find_completed_exp.sql (revision 18076) @@ -3,5 +3,6 @@ camera, telescope, - state + state, + NULL as epoch -- epoch FROM ( SELECT Index: /branches/cleanup/ippTools/share/regtool_pendingexp.sql =================================================================== --- /branches/cleanup/ippTools/share/regtool_pendingexp.sql (revision 18075) +++ /branches/cleanup/ippTools/share/regtool_pendingexp.sql (revision 18076) @@ -12,5 +12,6 @@ tess_id, end_stage, - label + label, + epoch FROM (SELECT Index: /branches/cleanup/ippTools/share/stacktool_find_complete_warps.sql =================================================================== --- /branches/cleanup/ippTools/share/stacktool_find_complete_warps.sql (revision 18075) +++ /branches/cleanup/ippTools/share/stacktool_find_complete_warps.sql (revision 18076) @@ -6,4 +6,6 @@ JOIN warpSkyfile USING(warp_id) + JOIN fakeRun + USING(fake_id) JOIN camRun USING(cam_id) Index: /branches/cleanup/ippTools/share/stacktool_inputskyfile.sql =================================================================== --- /branches/cleanup/ippTools/share/stacktool_inputskyfile.sql (revision 18075) +++ /branches/cleanup/ippTools/share/stacktool_inputskyfile.sql (revision 18076) @@ -11,10 +11,12 @@ JOIN warpRun ON stackInputSkyfile.warp_id = warpRun.warp_id -JOIN camProcessedExp - ON warpRun.cam_id = camProcessedExp.cam_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + USING(chip_id) JOIN chipProcessedImfile - ON chipRun.chip_id = chipProcessedImfile.chip_id + USING(chip_id) JOIN rawExp ON chipProcessedImfile.exp_id = rawExp.exp_id Index: /branches/cleanup/ippTools/share/warptool_exp.sql =================================================================== --- /branches/cleanup/ippTools/share/warptool_exp.sql (revision 18075) +++ /branches/cleanup/ippTools/share/warptool_exp.sql (revision 18076) @@ -1,8 +1,8 @@ SELECT - camProcessedExp.* + fakeRun.* FROM warpRun -JOIN camProcessedExp - USING(cam_id) +JOIN fakeRun + USING(fake_id) WHERE warpRun.state = 'run' - AND camProcessedExp.fault = 0 + AND fakeRun.state = 'stop' Index: /branches/cleanup/ippTools/share/warptool_imfile.sql =================================================================== --- /branches/cleanup/ippTools/share/warptool_imfile.sql (revision 18075) +++ /branches/cleanup/ippTools/share/warptool_imfile.sql (revision 18076) @@ -1,18 +1,26 @@ SELECT DISTINCT rawImfile.*, - warpRun.cam_id, + warpRun.fake_id, chipProcessedImfile.uri as chip_uri, chipProcessedImfile.path_base as chip_path_base, camProcessedExp.path_base as cam_path_base FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + ON camRun.chip_id = chipRun.chip_id JOIN chipProcessedImfile - ON camProcessedExp.chip_id = chipProcessedImfile.chip_id + ON chipRun.chip_id = chipProcessedImfile.chip_id JOIN rawImfile -- is there any reason not to refer back to rawimfiles? - ON chipProcessedImfile.exp_id = rawImfile.exp_id AND chipProcessedImfile.class_id = rawImfile.class_id + ON chipProcessedImfile.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id WHERE warpRun.state = 'run' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' Index: /branches/cleanup/ippTools/share/warptool_revertwarped.sql =================================================================== --- /branches/cleanup/ippTools/share/warptool_revertwarped.sql (revision 18075) +++ /branches/cleanup/ippTools/share/warptool_revertwarped.sql (revision 18076) @@ -1,7 +1,8 @@ DELETE FROM warpSkyfile -USING warpSkyfile, warpRun, camRun, chipRun, rawExp +USING warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp WHERE warpSkyfile.warp_id = warpRun.warp_id - AND warpRun.cam_id = camRun.cam_id + AND warpRun.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id AND camRun.chip_id = chipRun.chip_id AND chipRun.exp_id = rawExp.exp_id Index: /branches/cleanup/ippTools/share/warptool_scmap.sql =================================================================== --- /branches/cleanup/ippTools/share/warptool_scmap.sql (revision 18075) +++ /branches/cleanup/ippTools/share/warptool_scmap.sql (revision 18076) @@ -7,10 +7,18 @@ JOIN warpSkyCellMap USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) JOIN camProcessedExp - ON warpSkyCellMap.cam_id = camProcessedExp.cam_id + USING(cam_id) +JOIN chipRun + USING(chip_id) JOIN chipProcessedImfile - ON camProcessedExp.chip_id = chipProcessedImfile.chip_id + ON chipRun.chip_id = chipProcessedImfile.chip_id AND warpSkyCellMap.class_id = chipProcessedImfile.class_id --- move this as an option into the code --- WHERE +WHERE -- warpRun.state = 'run' + fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' Index: /branches/cleanup/ippTools/share/warptool_tooverlap.sql =================================================================== --- /branches/cleanup/ippTools/share/warptool_tooverlap.sql (revision 18075) +++ /branches/cleanup/ippTools/share/warptool_tooverlap.sql (revision 18076) @@ -1,5 +1,5 @@ SELECT warpRun.warp_id, - warpRun.cam_id, + warpRun.fake_id, warpRun.workdir, warpRun.tess_id, @@ -9,5 +9,7 @@ warpRun.magiced FROM warpRun -JOIN camProcessedExp +JOIN fakeRun + USING(fake_id) +JOIN camRun USING(cam_id) JOIN chipRun @@ -16,11 +18,12 @@ USING(exp_id) LEFT JOIN warpSkyCellMap - USING(warp_id, cam_id) + USING(warp_id) LEFT JOIN warpMask ON warpRun.label = warpMask.label WHERE warpRun.state = 'run' - AND camProcessedExp.fault = 0 + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' AND warpSkyCellMap.warp_id IS NULL - AND warpSkyCellMap.cam_id IS NULL AND warpMask.label IS NULL Index: /branches/cleanup/ippTools/share/warptool_towarped.sql =================================================================== --- /branches/cleanup/ippTools/share/warptool_towarped.sql (revision 18075) +++ /branches/cleanup/ippTools/share/warptool_towarped.sql (revision 18076) @@ -3,5 +3,6 @@ warpSkyCellMap.skycell_id, warpSkyCellMap.tess_id, - warpSkyCellMap.cam_id, + warpRun.fake_id, + camRun.cam_id, rawExp.camera, warpRun.workdir @@ -9,12 +10,14 @@ JOIN warpSkyCellMap USING(warp_id) -JOIN camProcessedExp - ON warpSkyCellMap.cam_id = camProcessedExp.cam_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + USING(chip_id) JOIN chipProcessedImfile - ON chipRun.chip_id = chipProcessedImfile.chip_id + USING(chip_id) JOIN rawExp - ON chipProcessedImfile.exp_id = rawExp.exp_id + ON chipRun.exp_id = rawExp.exp_id LEFT JOIN warpSkyfile ON warpRun.warp_id = warpSkyfile.warp_id @@ -25,4 +28,7 @@ WHERE warpRun.state = 'run' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' AND warpSkyfile.warp_id IS NULL AND warpSkyfile.skycell_id IS NULL Index: /branches/cleanup/ippTools/src/.cvsignore =================================================================== --- /branches/cleanup/ippTools/src/.cvsignore (revision 18075) +++ /branches/cleanup/ippTools/src/.cvsignore (revision 18076) @@ -23,4 +23,5 @@ difftool stacktool +faketool dettool detselect Index: /branches/cleanup/ippTools/src/Makefile.am =================================================================== --- /branches/cleanup/ippTools/src/Makefile.am (revision 18075) +++ /branches/cleanup/ippTools/src/Makefile.am (revision 18076) @@ -8,7 +8,8 @@ flatcorr \ magictool \ - pstamptool \ + pstamptool \ pxadmin \ pxinject \ + faketool \ pzgetexp \ pzgetimfiles \ @@ -28,4 +29,5 @@ pxconfig.h \ pxdata.h \ + pxfake.h \ pxio.h \ pxregister.h \ @@ -44,7 +46,8 @@ difftool.h \ flatcorr.h \ + faketool.h \ guidetool.h \ magictool.h \ - pstamptool.h \ + pstamptool.h \ pxinject.h \ pzgetexp.h \ @@ -65,4 +68,5 @@ pxdata.c \ pxerrors.c \ + pxfake.c \ pxfault.c \ pxio.c \ @@ -125,4 +129,10 @@ camtoolConfig.c +faketool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +faketool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +faketool_SOURCES = \ + faketool.c \ + faketoolConfig.c + magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) magictool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: /branches/cleanup/ippTools/src/camtool.c =================================================================== --- /branches/cleanup/ippTools/src/camtool.c (revision 18075) +++ /branches/cleanup/ippTools/src/camtool.c (revision 18076) @@ -27,5 +27,4 @@ #include "pxtools.h" -#include "pxwarp.h" #include "pxcam.h" #include "camtool.h" @@ -599,33 +598,32 @@ psFree(output); camProcessedExpRow *row = camProcessedExpRowAlloc( - pendingRow->cam_id, - pendingRow->chip_id, - uri, - bg, - bg_stdev, - bg_mean_stdev, - bias, - bias_stdev, - fringe_0, - fringe_1, - fringe_2, - sigma_ra, - sigma_dec, - ap_resid, - ap_resid_stdev, - zp_mean, - zp_stdev, - fwhm_major, - fwhm_minor, - dtime_detrend, - dtime_photom, - dtime_astrom, - hostname, - n_stars, - n_extended, - n_cr, - n_astrom, - path_base, - code + pendingRow->cam_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + bias, + bias_stdev, + fringe_0, + fringe_1, + fringe_2, + sigma_ra, + sigma_dec, + ap_resid, + ap_resid_stdev, + zp_mean, + zp_stdev, + fwhm_major, + fwhm_minor, + dtime_detrend, + dtime_photom, + dtime_astrom, + hostname, + n_stars, + n_extended, + n_cr, + n_astrom, + path_base, + code ); @@ -649,7 +647,7 @@ } - // should we stop here or proceed on to the warp stage? // NULL for end_stage means go as far as possible - // we can't start a warp run unlesss tess_id is defined + // XXX true for fake too? + // XXX we can't start a warp run unlesss tess_id is defined if ((pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam")) || (pendingRow->tess_id == NULL) || psStrcasestr(pendingRow->tess_id, "NULL")) { @@ -666,8 +664,10 @@ // else continue on... - if (!pxwarpQueueByCamID(config, + if (!pxfakeQueueByCamID(config, pendingRow->cam_id, pendingRow->workdir, pendingRow->label, + pendingRow->reduction, + pendingRow->expgroup, pendingRow->dvodb, pendingRow->tess_id, @@ -678,5 +678,5 @@ psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PS_ERR_UNKNOWN, false, "failed to queue new warpRun"); + psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); psFree(pendingRow); return false; Index: /branches/cleanup/ippTools/src/faketool.c =================================================================== --- /branches/cleanup/ippTools/src/faketool.c (revision 18075) +++ /branches/cleanup/ippTools/src/faketool.c (revision 18076) @@ -32,8 +32,8 @@ #include "faketool.h" -#include "camtool.h" static bool queueMode(pxConfig *config); static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); static bool pendingimfileMode(pxConfig *config); static bool addprocessedimfileMode(pxConfig *config); @@ -67,4 +67,5 @@ MODECASE(FAKETOOL_MODE_QUEUE, queueMode); MODECASE(FAKETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(FAKETOOL_MODE_PENDINGEXP, pendingexpMode); MODECASE(FAKETOOL_MODE_PENDINGIMFILE, pendingimfileMode); MODECASE(FAKETOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); @@ -171,5 +172,5 @@ // find the exp_id of all the exposures that we want to queue up. - psString query = pxDataGet("faketool_find_rawexp.sql"); + psString query = pxDataGet("faketool_find_camrun.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -179,6 +180,6 @@ if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); psFree(whereClause); } @@ -219,12 +220,12 @@ } - // loop over our list of exp_ids + // loop over our list of cam_ids for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; bool status; - psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + psS64 cam_id = psMetadataLookupS64(&status, md, "cam_id"); if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for cam_id"); psFree(output); return false; @@ -232,10 +233,10 @@ // queue the exp - if (!pxfakeQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!pxfakeQueueByCamID(config, cam_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); } psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue exp_id: %" PRId64, exp_id); + "failed to trying to queue cam_id: %" PRId64, cam_id); psFree(output); return false; @@ -354,4 +355,74 @@ +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_STR(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_STR(config->args, where, "-chip_id", "chip_id", "=="); + + psString query = pxDataGet("faketool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakePendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + static bool pendingimfileMode(pxConfig *config) { @@ -448,38 +519,17 @@ // fake_id, ext_tag, class_id are required - PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", true, false); - PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); - PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", true, false); + PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // optional - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); - PXOPT_LOOKUP_F32(bg, config->args, "-bg", false, false); - PXOPT_LOOKUP_F32(bg_stdev, config->args, "-bg_stdev", false, false); - PXOPT_LOOKUP_F32(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); - PXOPT_LOOKUP_F32(bias, config->args, "-bias", false, false); - PXOPT_LOOKUP_F32(bias_stdev, config->args, "-bias_stdev", false, false); - PXOPT_LOOKUP_F32(fringe_0, config->args, "-fringe_0", false, false); - PXOPT_LOOKUP_F32(fringe_1, config->args, "-fringe_1", false, false); - PXOPT_LOOKUP_F32(fringe_2, config->args, "-fringe_2", false, false); - PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); - PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); - PXOPT_LOOKUP_F32(ap_resid, config->args, "-ap_resid", false, false); - PXOPT_LOOKUP_F32(ap_resid_stdev, config->args, "-ap_resid_stdev", false, false); - PXOPT_LOOKUP_F32(zp_mean, config->args, "-zp_mean", false, false); - PXOPT_LOOKUP_F32(zp_stdev, config->args, "-zp_stdev", false, false); - PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); - PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); - PXOPT_LOOKUP_F32(dtime_detrend, config->args, "-dtime_detrend", false, false); - PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); - PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); - PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); - PXOPT_LOOKUP_F32(n_stars, config->args, "-n_stars", false, false); - PXOPT_LOOKUP_F32(n_extended, config->args, "-n_extended", false, false); - PXOPT_LOOKUP_F32(n_cr, config->args, "-n_cr", false, false); - PXOPT_LOOKUP_F32(n_astrom, config->args, "-n_astrom", false, false); - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + + PXOPT_LOOKUP_F32(dtime_fake, config->args, "-dtime_fake", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); // default values - PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); if (!psDBTransaction(config->dbh)) { @@ -493,30 +543,9 @@ class_id, uri, - bg, - bg_stdev, - bg_mean_stdev, - bias, - bias_stdev, - fringe_0, - fringe_1, - fringe_2, - sigma_ra, - sigma_dec, - ap_resid, - ap_resid_stdev, - zp_mean, - zp_stdev, - fwhm_major, - fwhm_minor, - dtime_detrend, - dtime_photom, - dtime_astrom, + dtime_fake, hostname, - n_stars, - n_extended, - n_cr, - n_astrom, path_base, - code + code, + NULL // epoch )) { // rollback @@ -748,8 +777,22 @@ PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); - if (!pxSetFaultCode(config->dbh, "fakeProcessedImfile", config->where, code)) { + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", false, false); + // convert fake_id into a psS64 + if (fake_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "fake_id", 0, "==", (psS64)atoll(fake_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fake_id"); + psFree(where); + return false; + } + } + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + if (!pxSetFaultCode(config->dbh, "fakeProcessedImfile", where, code)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); - return false; - } + psFree(where); + return false; + } + psFree(where); return true; @@ -984,10 +1027,8 @@ // camQueueFakeID() can only be run after fakeRun.state has been set to // stop - if (!pxcamQueueByFakeID(config, + if (!pxwarpQueueByFakeID(config, fakeRun->fake_id, fakeRun->workdir, fakeRun->label, - fakeRun->reduction, - fakeRun->expgroup, fakeRun->dvodb, fakeRun->tess_id, Index: /branches/cleanup/ippTools/src/faketool.h =================================================================== --- /branches/cleanup/ippTools/src/faketool.h (revision 18076) +++ /branches/cleanup/ippTools/src/faketool.h (revision 18076) @@ -0,0 +1,46 @@ +/* + * faketool.h + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef FAKETOOL_H +#define FAKETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + FAKETOOL_MODE_NONE = 0x0, + FAKETOOL_MODE_QUEUE, + FAKETOOL_MODE_UPDATERUN, + FAKETOOL_MODE_PENDINGIMFILE, + FAKETOOL_MODE_PENDINGEXP, + FAKETOOL_MODE_ADDPROCESSEDIMFILE, + FAKETOOL_MODE_PROCESSEDIMFILE, + FAKETOOL_MODE_REVERTPROCESSEDIMFILE, + FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, + FAKETOOL_MODE_BLOCK, + FAKETOOL_MODE_MASKED, + FAKETOOL_MODE_UNMASKED, + FAKETOOL_MODE_UNBLOCK, + FAKETOOL_MODE_RETRYPROCESSEDIMFILE, + FAKETOOL_MODE_PENDINGCLEANUP, + FAKETOOL_MODE_DONECLEANUP, +} FAKETOOLMode; + +pxConfig *faketoolConfig(pxConfig *config, int argc, char **argv); + +#endif // FAKETOOL_H Index: /branches/cleanup/ippTools/src/faketoolConfig.c =================================================================== --- /branches/cleanup/ippTools/src/faketoolConfig.c (revision 18075) +++ /branches/cleanup/ippTools/src/faketoolConfig.c (revision 18076) @@ -232,4 +232,15 @@ + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chiptool ID", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + // -pendingimfile psMetadata *pendingimfileArgs = psMetadataAlloc(); @@ -416,15 +427,18 @@ PXOPT_ADD_MODE("-queue", "create runs from raw exposures", - FAKETOOL_MODE_QUEUE, queueArgs); + FAKETOOL_MODE_QUEUE, queueArgs); PXOPT_ADD_MODE("-updaterun", "change fake run properties", - FAKETOOL_MODE_UPDATERUN, updaterunArgs); + FAKETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exposures", + FAKETOOL_MODE_PENDINGEXP, pendingexpArgs); PXOPT_ADD_MODE("-pendingimfile", "show pending imfiles", - FAKETOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + FAKETOOL_MODE_PENDINGIMFILE, pendingimfileArgs); PXOPT_ADD_MODE("-addprocessedimfile", "add a processed imfile", - FAKETOOL_MODE_ADDPROCESSEDIMFILE,addprocessedimfileArgs); + FAKETOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); PXOPT_ADD_MODE("-processedimfile", "show processed imfiles", - FAKETOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); - PXOPT_ADD_MODE("-updateprocessedimfile", "change procesed imfile properties", FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); - PXOPT_ADD_MODE("-revertprocessedimfile", "undo a processed imfile", FAKETOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); + FAKETOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); + PXOPT_ADD_MODE("-revertprocessedimfile", "undo a processed imfile", + FAKETOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); PXOPT_ADD_MODE("-block", "set a label block", FAKETOOL_MODE_BLOCK, blockArgs); PXOPT_ADD_MODE("-masked", "show blocked labels", FAKETOOL_MODE_MASKED, maskedArgs); Index: /branches/cleanup/ippTools/src/pxfake.c =================================================================== --- /branches/cleanup/ippTools/src/pxfake.c (revision 18076) +++ /branches/cleanup/ippTools/src/pxfake.c (revision 18076) @@ -0,0 +1,188 @@ +/* + * pxfake.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxcam.h" + +bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid fakeRun state: %s", state); + return false; + } + + char *query = "UPDATE fakeRun SET state = '%s' WHERE fake_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, fake_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for fake_id %" PRId64, fake_id); + return false; + } + + return true; +} + + +bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid fakeRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE fakeRun SET fakeRun.label = '%s' WHERE fake_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, label, fake_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for fake_id %" PRId64, fake_id); + return false; + } + + return true; +} + + +bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET fakeRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +psS64 pxfakeQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *reduction, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psTrace("pxtool", PS_LOG_INFO, "attemping to queue cam_id: %"PRId64, cam_id); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("faketool_queue_cam_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQuery(config->dbh, query, + "run", // state + workdir ? workdir : "NULL", + label ? label : "NULL", + reduction? reduction: "NULL", + expgroup ? expgroup : "NULL", + dvodb ? dvodb : "NULL", + tess_id ? tess_id : "NULL", + end_stage ? end_stage : "NULL", + (long long)cam_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have changed just one row"); + return false; + } + + return psDBLastInsertID(config->dbh); +} Index: /branches/cleanup/ippTools/src/pxfake.h =================================================================== --- /branches/cleanup/ippTools/src/pxfake.h (revision 18076) +++ /branches/cleanup/ippTools/src/pxfake.h (revision 18076) @@ -0,0 +1,44 @@ +/* + * pxfake.h + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXFAKE_H +#define PXFAKE_H 1 + +#include + +#include "pxtools.h" + +bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state); +bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label); +bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + + +psS64 pxfakeQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *reduction, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage); + + +#endif // PXFAKE_H Index: /branches/cleanup/ippTools/src/pxinject.c =================================================================== --- /branches/cleanup/ippTools/src/pxinject.c (revision 18075) +++ /branches/cleanup/ippTools/src/pxinject.c (revision 18076) @@ -91,9 +91,9 @@ if (!newExpInsert(config->dbh, - 0x0, // exp_id + 0x0, // exp_id tmp_exp_name, tmp_camera, tmp_telescope, - "reg", // state + "reg", // state workdir, "dirty", @@ -101,6 +101,7 @@ dvodb, tess_id, - end_stage, - label + end_stage, + label, + NULL // epoch ) ) { @@ -141,5 +142,5 @@ // insert with error flag state set to 0 (no errors) - if (!newImfileInsert(config->dbh, (psS64)atoll(exp_id), tmp_class_id, uri)) { + if (!newImfileInsert(config->dbh, (psS64)atoll(exp_id), tmp_class_id, uri, NULL)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; Index: /branches/cleanup/ippTools/src/pxtools.h =================================================================== --- /branches/cleanup/ippTools/src/pxtools.h (revision 18075) +++ /branches/cleanup/ippTools/src/pxtools.h (revision 18076) @@ -28,13 +28,16 @@ #include +#include "pxconfig.h" #include "pxtoolsErrorCodes.h" + +#include "pxcam.h" +#include "pxchip.h" +#include "pxdata.h" +#include "pxfake.h" +#include "pxwarp.h" #include "pxio.h" -#include "pxdata.h" +#include "pxregister.h" #include "pxtag.h" -#include "pxregister.h" -#include "pxchip.h" -#include "pxcam.h" #include "pxtree.h" -#include "pxconfig.h" # define MAX_ROWS 10e9 Index: /branches/cleanup/ippTools/src/pxwarp.c =================================================================== --- /branches/cleanup/ippTools/src/pxwarp.c (revision 18075) +++ /branches/cleanup/ippTools/src/pxwarp.c (revision 18076) @@ -135,6 +135,6 @@ -bool pxwarpQueueByCamID(pxConfig *config, - psS64 cam_id, +bool pxwarpQueueByFakeID(pxConfig *config, + psS64 fake_id, char *workdir, char *label, @@ -148,5 +148,5 @@ if (!warpRunInsert(config->dbh, 0, // ID - cam_id, + fake_id, "warp", // mode "run", // state Index: /branches/cleanup/ippTools/src/pxwarp.h =================================================================== --- /branches/cleanup/ippTools/src/pxwarp.h (revision 18075) +++ /branches/cleanup/ippTools/src/pxwarp.h (revision 18076) @@ -30,6 +30,6 @@ bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); -bool pxwarpQueueByCamID(pxConfig *config, - psS64 cam_id, +bool pxwarpQueueByFakeID(pxConfig *config, + psS64 fake_id, char *workdir, char *label, Index: /branches/cleanup/ippTools/src/pzgetexp.c =================================================================== --- /branches/cleanup/ippTools/src/pzgetexp.c (revision 18075) +++ /branches/cleanup/ippTools/src/pzgetexp.c (revision 18076) @@ -192,6 +192,7 @@ " SElECT" " incoming.*," - " NULL," - " 0" + " NULL," // imfiles + " 0," // fault + " NULL" // epoch " FROM incoming" " LEFT JOIN summitExp" Index: /branches/cleanup/ippTools/src/pzgetimfiles.c =================================================================== --- /branches/cleanup/ippTools/src/pzgetimfiles.c (revision 18075) +++ /branches/cleanup/ippTools/src/pzgetimfiles.c (revision 18076) @@ -250,5 +250,6 @@ " incoming.class," " incoming.class_id," - " incoming.uri" + " incoming.uri," + " NULL" // epoch " FROM incoming"; if (!p_psDBRunQuery(config->dbh, query)) { @@ -307,5 +308,6 @@ " incoming.camera," " incoming.telescope," - " \"run\"" + " \"run\"," // state + " NULL" " FROM incoming" " LEFT JOIN pzDownloadExp" Index: /branches/cleanup/ippTools/src/pztool.c =================================================================== --- /branches/cleanup/ippTools/src/pztool.c (revision 18075) +++ /branches/cleanup/ippTools/src/pztool.c (revision 18076) @@ -111,5 +111,6 @@ camera, telescope, - uri + uri, + NULL // epoch )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -370,5 +371,6 @@ class_id, uri, - code + code, + NULL // epoch )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -484,4 +486,10 @@ pzDownloadExpRow *doneExp = pzDownloadExpObjectFromMetadata(row); + if (!doneExp) { + psError(PS_ERR_UNKNOWN, false, "pzDownloadExpObjectFromMetadata() failed"); + psFree(doneExp); + psFree(output); + return false; + } if (!newExpInsert(config->dbh, @@ -497,5 +505,6 @@ tess_id, // tess_id end_stage, // end_stage - label + label, + NULL // epoch ) ) { @@ -515,5 +524,6 @@ " %" PRId64 "," // exp_id " pzDownloadImfile.class_id," // tmp_class_id - " pzDownloadImfile.uri" // uri + " pzDownloadImfile.uri," // uri + " NULL" // epoch " FROM pzDownloadImfile" " WHERE" Index: /branches/cleanup/ippTools/src/regtool.c =================================================================== --- /branches/cleanup/ippTools/src/regtool.c (revision 18075) +++ /branches/cleanup/ippTools/src/regtool.c (revision 18076) @@ -280,5 +280,6 @@ object, hostname, - code + code, + NULL )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -630,6 +631,11 @@ psMetadata *row = output->data[0]; // convert metadata into a newExp object + psMetadataConfigPrint(stdout, row); newExpRow *newExp = newExpObjectFromMetadata(row); psFree(output); + if (!newExp) { + psError(PS_ERR_UNKNOWN, false, "this should not happen"); + return false; + } // carry through these values @@ -700,5 +706,6 @@ 0.0, hostname, - code + code, + NULL )) { // rollback Index: /branches/cleanup/ippTools/src/warptool.c =================================================================== --- /branches/cleanup/ippTools/src/warptool.c (revision 18075) +++ /branches/cleanup/ippTools/src/warptool.c (revision 18076) @@ -111,5 +111,5 @@ // required options - PXOPT_LOOKUP_STR(cam_id, config->args, "-cam_id", true, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", true, false); PXOPT_LOOKUP_STR(mode, config->args, "-mode", true, false); // check mode @@ -133,5 +133,5 @@ warpRunRow *warpRun = warpRunRowAlloc( 0, // ID - (psS64)atoll(cam_id), + (psS64)atoll(fake_id), mode, "run", // state @@ -176,5 +176,5 @@ PXOPT_LOOKUP_STR(warp_id, config->args, "-warp_id", false, false); - PXOPT_LOOKUP_STR(cam_id, config->args, "-cam_id", false, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", false, false); PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); @@ -190,8 +190,8 @@ } - // convert cam_id into a psS64 - if (cam_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "cam_id", 0, "==", (psS64)atoll(cam_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item cam_id"); + // convert fake_id into a psS64 + if (fake_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "fake_id", 0, "==", (psS64)atoll(fake_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fake_id"); psFree(where); return false; @@ -526,5 +526,4 @@ PXOPT_LOOKUP_STR(mapfile, config->args, "-mapfile", false, false); PXOPT_LOOKUP_STR(warp_id, config->args, "-warp_id", false, false); - PXOPT_LOOKUP_STR(cam_id, config->args, "-cam_id", false, false); PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); @@ -544,5 +543,11 @@ } } else { - warpSkyCellMapInsert(config->dbh, (psS64)atoll(warp_id), NULL, NULL, 0, NULL, code); + warpSkyCellMapInsert(config->dbh, + (psS64)atoll(warp_id), + NULL, // skycell_id + NULL, // tess_id + NULL, // class_id + code // fault + ); } @@ -1029,5 +1034,5 @@ PXOPT_LOOKUP_STR(warp_id, config->args, "-warp_id", false, false); - PXOPT_LOOKUP_STR(cam_id, config->args, "-cam_id", false, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", false, false); PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); @@ -1046,8 +1051,8 @@ PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); - // convert cam_id into a psS64 - if (cam_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "camRun.cam_id", 0, "==", (psS64)atoll(cam_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item cam_id"); + // convert fake_id into a psS64 + if (fake_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "fakeRun.fake_id", 0, "==", (psS64)atoll(fake_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fake_id"); psFree(where); return false; Index: /branches/cleanup/ippTools/src/warptoolConfig.c =================================================================== --- /branches/cleanup/ippTools/src/warptoolConfig.c (revision 18075) +++ /branches/cleanup/ippTools/src/warptoolConfig.c (revision 18076) @@ -49,5 +49,5 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-fake_id", 0, "define camtool ID (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-mode", 0, @@ -72,10 +72,10 @@ // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - // XXX need to allow multiple cam_ids + // XXX need to allow multiple fake_ids psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); - // XXX need to allow multiple cam_ids - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, - "search by cam_id", NULL); + // XXX need to allow multiple fake_ids + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by fake_id", NULL); // XXX need to allow multiple chip_ids psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, @@ -176,5 +176,5 @@ psMetadataAddStr(expArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); - psMetadataAddStr(expArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(expArgs, PS_LIST_TAIL, "-fake_id", 0, "search by camtool ID", NULL); psMetadataAddU64(expArgs, PS_LIST_TAIL, "-limit", 0, @@ -187,5 +187,5 @@ psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); - psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by camtool ID", NULL); psMetadataAddU64(imfileArgs, PS_LIST_TAIL, "-limit", 0, @@ -279,5 +279,5 @@ psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure tag", NULL); - psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "define phase 3 version of exposure tag", NULL); psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, @@ -288,5 +288,5 @@ // -revertwarped psMetadata *revertwarpedArgs = psMetadataAlloc(); - // XXX need to allow multiple cam_ids + // XXX need to allow multiple fake_ids psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); @@ -295,7 +295,7 @@ psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, "searcy by tessellation ID", NULL); - // XXX need to allow multiple cam_ids - psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-cam_id", 0, - "search by cam_id", NULL); + // XXX need to allow multiple fake_ids + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by fake_id", NULL); // XXX need to allow multiple chip_ids psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-chip_id", 0,