Index: branches/eam_branches/ipp-20110906/ippTools/share/Makefile.am =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/Makefile.am (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/Makefile.am (revision 32630) @@ -76,4 +76,6 @@ chiptool_definecopy.sql \ chiptool_donecleanup.sql \ + chiptool_dropprocessedimfile_update.sql \ + chiptool_dropprocessedimfile_delete.sql \ chiptool_find_rawexp.sql \ chiptool_listrun.sql \ @@ -275,4 +277,5 @@ magicdstool_definebyquery_warp_bg.sql \ magicdstool_definebyquery_diff.sql \ + magicdstool_definecopy_camera.sql \ magicdstool_definecopy_chip.sql \ magicdstool_definecopy_warp.sql \ @@ -424,4 +427,10 @@ laptool_pendingexp.sql \ laptool_pendingrun.sql \ - laptool_stacks.sql + laptool_WSdiff_check.sql \ + laptool_stacks.sql \ + vptool_find_rawexp.sql \ + vptool_pendingimfile.sql \ + vptool_pendingrun.sql \ + vptool_processedcell.sql \ + vptool_revertrun.sql Index: branches/eam_branches/ipp-20110906/ippTools/share/camtool_find_pendingimfile.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/camtool_find_pendingimfile.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/camtool_find_pendingimfile.sql (revision 32630) @@ -13,7 +13,9 @@ USING (chip_id) JOIN chipProcessedImfile - USING (chip_id) + USING (chip_id, exp_id) JOIN rawExp - ON chipProcessedImfile.exp_id = rawExp.exp_id + USING(exp_id) +JOIN rawImfile + USING(exp_id, class_id, exp_time) LEFT JOIN camProcessedExp USING(cam_id) @@ -22,2 +24,3 @@ WHERE camMask.label IS NULL + AND rawImfile.ignored = 0 Index: branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_delete.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_delete.sql (revision 32630) +++ branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_delete.sql (revision 32630) @@ -0,0 +1,6 @@ +DELETE FROM magicDSFile + USING magicDSRun, magicDSFile, chipRun, chipProcessedImfile +WHERE chipRun.chip_id = chipProcessedImfile.chip_id + AND magicDSRun.stage ='chip' and magicDSRun.stage_id = chipRun.chip_id + AND magicDSFile.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = chipProcessedImfile.class_id Index: branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_update.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_update.sql (revision 32630) +++ branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_update.sql (revision 32630) @@ -0,0 +1,5 @@ +UPDATE chipProcessedImfile + SET quality = %d, + data_state = 'full', + magicked = 0, + fault = 0 Index: branches/eam_branches/ipp-20110906/ippTools/share/laptool_WSdiff_check.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/laptool_WSdiff_check.sql (revision 32630) +++ branches/eam_branches/ipp-20110906/ippTools/share/laptool_WSdiff_check.sql (revision 32630) @@ -0,0 +1,24 @@ +SELECT warp_id,warp_state,W.skycell_id,stack_id,stack_state,S.seq_id + FROM +( SELECT warp_id,skycell_id,seq_id,warpRun.state AS warp_state + FROM warpSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + JOIN lapExp USING(exp_id,chip_id) + JOIN lapRun USING(lap_id) + WHERE 1 + @WHERE@ +) AS W +LEFT JOIN +( SELECT stack_id,skycell_id,seq_id,stackRun.state AS stack_state + FROM stackRun + JOIN stackAssociationMap USING(stack_id) + JOIN lapRun ON (stackAssociationMap.sass_id = lapRun.final_sass_id) + ) AS S +ON ( (W.seq_id = S.seq_id) + AND (W.skycell_id = S.skycell_id) ) + + Index: branches/eam_branches/ipp-20110906/ippTools/share/laptool_definerun.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/laptool_definerun.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/laptool_definerun.sql (revision 32630) @@ -1,3 +1,8 @@ -SELECT DISTINCT want.exp_id, have.chip_id, false as private, true as active, false as pairwise +SELECT DISTINCT + want.exp_id, + have.chip_id, + COALESCE(have.private,false) as private, + true as active, + false as pairwise FROM (SELECT exp_id FROM rawExp @@ -8,15 +13,21 @@ ) AS want LEFT JOIN - (SELECT exp_id,MAX(chip_id) AS chip_id, chipRun.state + (SELECT + exp_id, + MAX(chip_id) AS chip_id, + chipRun.state, + lapExp.private FROM lapExp JOIN chipRun USING(exp_id,chip_id) - where private IS FALSE - AND chip_id IS NOT NULL + JOIN lapRun USING(lap_id) + where chip_id IS NOT NULL AND - ( +-- This is the restriction to only draw from the current lapSequence + @CHIPWHERE@ +-- ( -- (active = TRUE) OR - (chipRun.state = 'full') OR - (chipRun.state = 'new') + -- (chipRun.state = 'full') OR + -- (chipRun.state = 'new') -- when we can do updates, put that here. - ) +-- ) GROUP BY exp_id ) AS have USING(exp_id) Index: branches/eam_branches/ipp-20110906/ippTools/share/laptool_pendingexp.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/laptool_pendingexp.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/laptool_pendingexp.sql (revision 32630) @@ -3,7 +3,9 @@ lapRun.dist_group, lapRun.registered, lapRun.fault, lapRun.quick_sass_id, lapRun.final_sass_id, exp_id, chip_id, pair_id, private, pairwise, active, lapExp.data_state, - dateobs, object, comment + dateobs, object, comment, + chipRun.state AS chip_state FROM lapRun JOIN lapExp USING(lap_id) JOIN rawExp USING(exp_id,filter) + LEFT JOIN chipRun USING(exp_id,chip_id) WHERE active IS TRUE AND lapRun.fault = 0 -- lap_id restriction here. Index: branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_definecopy_camera.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_definecopy_camera.sql (revision 32630) +++ branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_definecopy_camera.sql (revision 32630) @@ -0,0 +1,31 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + chipRun.exp_id, + 'camera' AS stage, + camRun.cam_id AS stage_id, + camRun.data_group, + camRun.cam_id, + camRun.label, + CAST(NULL AS SIGNED) AS inv_magic_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun AS oldWarpRun + ON (!magicRun.inverse AND diffInputSkyfile.warp1 = oldWarpRun.warp_id) + OR ( magicRun.inverse AND diffInputSkyfile.warp2 = oldWarpRun.warp_id) + -- This ends the usual JOIN down to the raw level + JOIN rawExp + ON rawExp.exp_id = magicRun.exp_id + JOIN chipRun + ON chipRun.exp_id = rawExp.exp_id + JOIN camRun + ON camRun.chip_id = chipRun.chip_id + LEFT JOIN magicDSRun + ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'camera' + AND magicDSRun.label = '%s' +WHERE magicRun.state = 'full' + -- HOOK: NEWLINE HERE TO ACTIVATE %s AND magicDSRun.magic_ds_id IS NULL + AND camRun.magicked = 0 + AND camRun.state = 'full' Index: branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_camera.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_camera.sql (revision 32630) @@ -14,4 +14,5 @@ stage_id, 'exposure' AS component, + 0 AS mismatched_tess, CAST(NULL AS CHAR(255)) AS uri, camProcessedExp.path_base, Index: branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_chip.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_chip.sql (revision 32630) @@ -14,4 +14,5 @@ stage_id, class_id AS component, + 0 AS mismatched_tess, chipProcessedImfile.uri, chipProcessedImfile.path_base, Index: branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_diff.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_diff.sql (revision 32630) @@ -15,4 +15,5 @@ diffRun.magicked, diffSkyfile.skycell_id AS component, + 0 AS mismatched_tess, CAST(NULL AS CHAR(255)) AS uri, diffSkyfile.path_base, @@ -63,4 +64,5 @@ diffRun.magicked, diffSkyfile.skycell_id AS component, + 0 AS mismatched_tess, CAST(NULL AS CHAR(255)) AS uri, diffSkyfile.path_base, Index: branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_raw.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_raw.sql (revision 32630) @@ -14,4 +14,5 @@ rawExp.magicked, class_id as component, + 0 AS mismatched_tess, rawImfile.uri AS uri, -- XXX: replace this with rawImfile.path_base once it exists Index: branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_warp.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_warp.sql (revision 32630) @@ -14,4 +14,6 @@ warpRun.magicked, warpSkyfile.skycell_id as component, + diffRun.tess_id != warpRun.tess_id AS mismatched_tess, + diffRun.tess_id as diff_tess_id, warpSkyfile.uri, warpSkyfile.path_base, @@ -27,4 +29,5 @@ JOIN magicMask USING (magic_id) JOIN magicRun USING (magic_id) +JOIN diffRun USING(diff_id) JOIN warpRun ON warp_id = stage_id JOIN warpSkyfile USING(warp_id) Index: branches/eam_branches/ipp-20110906/ippTools/share/pubtool_definerun.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/pubtool_definerun.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/pubtool_definerun.sql (revision 32630) @@ -15,4 +15,5 @@ JOIN diffRun JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile using(diff_id) JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 -- Only JOINing input, not reference! JOIN fakeRun USING(fake_id) @@ -30,4 +31,5 @@ ) ) + AND diffSkyfile.quality = 0 -- WHERE hook %s UNION Index: branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_create_tables.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_create_tables.sql (revision 32630) @@ -554,6 +554,5 @@ KEY(workdir_state), KEY(label), - KEY(data_group), - INDEX(add_id, cam_id) + KEY(data_group) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1157,6 +1156,6 @@ tess_id VARCHAR(64), diff_skyfile_id BIGINT AUTO_INCREMENT, + PRIMARY KEY(diff_skyfile_id), KEY(diff_id, skycell_id), - KEY(diff_skyfile_id), KEY(warp1), KEY(warp2), @@ -1969,9 +1968,11 @@ dtime_verify FLOAT, dtime_merge FLOAT, - dtime_script, - epoch TIMESTAMP, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, mergedvodb_path VARCHAR(255) NOT NULL, fault SMALLINT NOT NULL, - KEY(minidvodb_id), CONSTRAINT UNIQUE(minidvodb_id), FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun (minidvodb_id) + UNIQUE KEY (minidvodb_id), + KEY(minidvodb_id), + FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1990,7 +1991,39 @@ FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; - - - +CREATE TABLE vpRun ( + vp_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + label VARCHAR(64), + data_group VARCHAR(64), + workdir VARCHAR(255), + note VARCHAR(255), + dest_id BIGINT, + outroot VARCHAR(255), + dtime_script FLOAT, + hostname VARCHAR(64), + fault SMALLINT NOT NULL DEFAULT 0, + PRIMARY KEY(vp_id), + KEY(exp_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(fault), + INDEX(vp_id, exp_id), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE vpProcessedCell ( + vp_id BIGINT, + class_id VARCHAR(64), + cell_id VARCHAR(64), + dtime_photom FLOAT, + quality SMALLINT NOT NULL DEFAULT 0, + path_base VARCHAR(255), + fault SMALLINT NOT NULL DEFAULT 0, + KEY(quality), + KEY(fault), + FOREIGN KEY(vp_id) REFERENCES vpRun(vp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; -- This comment line is here to avoid empty query error. Index: branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_drop_tables.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_drop_tables.sql (revision 32630) @@ -97,4 +97,12 @@ DROP TABLE IF EXISTS diffPhotRun; DROP TABLE IF EXISTS diffPhotSkyfile; +DROP TABLE IF EXISTS lapExp; +DROP TABLE IF EXISTS lapRun; +DROP TABLE IF EXISTS lapSequence; +DROP TABLE IF EXISTS minidvodbRun; +DROP TABLE IF EXISTS minidvodbCopy; +DROP TABLE IF EXISTS minidvodbProcessed; +DROP TABLE IF EXISTS vpProcessedCell; +DROP TABLE IF EXISTS vpRun; SET FOREIGN_KEY_CHECKS=1 Index: branches/eam_branches/ipp-20110906/ippTools/share/vptool_find_rawexp.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/vptool_find_rawexp.sql (revision 32630) +++ branches/eam_branches/ipp-20110906/ippTools/share/vptool_find_rawexp.sql (revision 32630) @@ -0,0 +1,12 @@ +-- this query is used to find potental rawExps to be queued for vptool +-- processeing +SELECT DISTINCT + rawExp.*, + newExp.label +FROM rawExp +JOIN newExp using (exp_id) +JOIN rawImfile using(exp_id) +LEFT JOIN vpRun USING (exp_id) +WHERE + rawExp.fault = 0 + AND rawImfile.video_cells Index: branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingimfile.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingimfile.sql (revision 32630) +++ branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingimfile.sql (revision 32630) @@ -0,0 +1,12 @@ +SELECT + vpRun.*, + rawImfile.class_id, + rawImfile.uri +FROM vpRun + JOIN rawExp USING(exp_id) + JOIN rawImfile USING(exp_id) + LEFT JOIN vpProcessedCell USING(vp_id, class_id) +WHERE rawImfile.video_cells + AND vpRun.state = 'new' + AND vpProcessedCell.vp_id IS NULL + Index: branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingrun.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingrun.sql (revision 32630) +++ branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingrun.sql (revision 32630) @@ -0,0 +1,17 @@ +SELECT vp_id, + rawExp.camera, + CONCAT_WS('.', exp_name, exp_id) AS exp_tag, + vpRun.label, + vpRun.workdir, + vpRun.dest_id, + rcDestination.name AS dest_name, + rcDestination.dbname AS ds_dbname, + rcDestination.dbhost AS ds_dbhost, + IFNULL(Label.priority, 10000) as priority +FROM vpRun + JOIN rawExp USING(exp_id) + LEFT JOIN rcDestination USING(dest_id) + LEFT JOIN Label ON vpRun.label = Label.label +WHERE vpRun.state = 'new' + AND vpRun.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: branches/eam_branches/ipp-20110906/ippTools/share/vptool_processedcell.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/vptool_processedcell.sql (revision 32630) +++ branches/eam_branches/ipp-20110906/ippTools/share/vptool_processedcell.sql (revision 32630) @@ -0,0 +1,22 @@ +SELECT + vpProcessedCell.*, + vpRun.state, + vpRun.label, + vpRun.data_group, + vpRun.outroot, + rawExp.exp_id, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.filter, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel, + rawExp.dateobs, + rawExp.obs_mode, + rawExp.comment, + rawExp.ra, + rawExp.decl, + rawExp.exp_time +FROM vpRun + JOIN vpProcessedCell USING(vp_id) + JOIN rawExp USING(exp_id) Index: branches/eam_branches/ipp-20110906/ippTools/share/vptool_revertrun.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/vptool_revertrun.sql (revision 32630) +++ branches/eam_branches/ipp-20110906/ippTools/share/vptool_revertrun.sql (revision 32630) @@ -0,0 +1,4 @@ +UPDATE vpRun + SET fault = 0 +WHERE vpRun.state = 'new' + AND fault > 0 Index: branches/eam_branches/ipp-20110906/ippTools/share/warptool_scmap.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/warptool_scmap.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/warptool_scmap.sql (revision 32630) @@ -29,4 +29,5 @@ ON chipRun.chip_id = chipProcessedImfile.chip_id AND warpSkyCellMap.class_id = chipProcessedImfile.class_id + AND chipProcessedImfile.quality = 0 JOIN rawImfile ON chipRun.exp_id = rawImfile.exp_id Index: branches/eam_branches/ipp-20110906/ippTools/share/warptool_towarped.sql =================================================================== --- branches/eam_branches/ipp-20110906/ippTools/share/warptool_towarped.sql (revision 32363) +++ branches/eam_branches/ipp-20110906/ippTools/share/warptool_towarped.sql (revision 32630) @@ -100,4 +100,7 @@ AND warpSkyfile.fault = 0 AND camRun.state = 'full' + AND chipProcessedImfile.quality = 0 + -- Maybe add this to solve the if any chip is bad a whole warp can't be updated problem? + -- AND (chipProcessedImfile.fault = 0 OR chipProcessedImfile.fault = 26) AND (Label.active OR Label.active IS NULL) -- where hook 2 %s