Index: /branches/pap_branches/pap_branch_20081109/dbconfig/.cvsignore =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/.cvsignore (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/.cvsignore (revision 22235) @@ -0,0 +1,2 @@ +ippdb.m4 +ippdb.mdc Index: /branches/pap_branches/pap_branch_20081109/dbconfig/Makefile =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/Makefile (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/Makefile (revision 22235) @@ -0,0 +1,26 @@ +SHELL=/bin/sh +GLUEFORGE=`which glueforge` + +all: ippdb.mdc + +ippdb.mdc : ipp.m4 *.md + m4 ipp.m4 > ippdb.mdc + +## please leave the output target as ippdb.src and +## move to ippdb by hand for cvs import +install: ippdb.mdc + rm -rf ../ippdb.src + $(GLUEFORGE) -i ippdb.mdc --output ../ippdb.src + chmod +x ../ippdb.src/autogen.sh + +src: ippdb.mdc + $(GLUEFORGE) -i ippdb.mdc --output ../ippdb + chmod +x ../ippdb/autogen.sh + +foo: src + $(MAKE) -C ../ippdb install + +build : ippdb ippdb.mdc + +clean: + @rm -f *~ Index: /branches/pap_branches/pap_branch_20081109/dbconfig/calibration.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/calibration.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/calibration.md (revision 22235) @@ -0,0 +1,15 @@ + +# table of the DVO databases which get calibrated +calDB METADATA + cal_id S64 0 # Primary Key AUTO_INCREMENT + dvodb STR 64 + state STR 64 +END + +calRun METADATA + cal_id S64 0 # Primary Key AUTO_INCREMENT + region STR 64 # success or failure + last_step STR 64 # fkey(cal_id) ref calDB(cal_id) + state STR 64 # +END + Index: /branches/pap_branches/pap_branch_20081109/dbconfig/cam.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/cam.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/cam.md (revision 22235) @@ -0,0 +1,93 @@ +camRun METADATA + cam_id S64 0 # Primary Key AUTO_INCREMENT + chip_id S64 0 # Key INDEX(cam_id, chip_id) fkey(chip_id) ref chipRun(chip_id) + state STR 64 # key + workdir STR 255 + workdir_state STR 64 # key + label STR 64 # key + reduction STR 64 + expgroup STR 64 # key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key +END + +camProcessedExp METADATA +# the camPendingExp row gets deleted so we can not put a fkey on cam_id + cam_id S64 0 # Primary Key + uri STR 255 # fkey(cam_id) ref camRun(cam_id) + + bg F32 0.0 + bg_stdev F32 0.0 + bg_mean_stdev F32 0.0 + bias F32 0.0 + bias_stdev F32 0.0 + fringe_0 F32 0.0 + fringe_1 F32 0.0 + fringe_2 F32 0.0 + sigma_ra F32 0.0 + sigma_dec F32 0.0 + ap_resid F32 0.0 + ap_resid_stdev F32 0.0 + + zpt_obs F32 0.0 + zpt_stdev F32 0.0 + zpt_lq F32 0.0 + zpt_uq F32 0.0 + + fwhm_major F32 0.0 + fwhm_major_lq F32 0.0 + fwhm_major_uq F32 0.0 + + fwhm_minor F32 0.0 + fwhm_minor_lq F32 0.0 + fwhm_minor_uq F32 0.0 + + iq_fwhm_major F32 0.0 + iq_fwhm_major_err F32 0.0 + iq_fwhm_minor F32 0.0 + iq_fwhm_minor_err F32 0.0 + + iq_m2 F32 0.0 + iq_m2_err F32 0.0 + iq_m2_lq F32 0.0 + iq_m2_uq F32 0.0 + + iq_m2c F32 0.0 + iq_m2c_err F32 0.0 + iq_m2c_lq F32 0.0 + iq_m2c_uq F32 0.0 + + iq_m2s F32 0.0 + iq_m2s_err F32 0.0 + iq_m2s_lq F32 0.0 + iq_m2s_uq F32 0.0 + + iq_m3 F32 0.0 + iq_m3_err F32 0.0 + iq_m3_lq F32 0.0 + iq_m3_uq F32 0.0 + + iq_m4 F32 0.0 + iq_m4_err F32 0.0 + iq_m4_lq F32 0.0 + iq_m4_uq F32 0.0 + + dtime_script F32 0.0 + dtime_astrom F32 0.0 + dtime_addstar F32 0.0 + + hostname STR 64 + n_stars S32 0 + n_psfstars S32 0 + n_iqstars S32 0 + n_extended S32 0 + n_cr S32 0 + n_astrom S32 0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +camMask METADATA + label STR 64 # Primary Key +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/changes.txt =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/changes.txt (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/changes.txt (revision 22235) @@ -0,0 +1,687 @@ +-- How to manually change the database between versions via mySQL commands. +-- +-- This file is generated manually, and may not be complete. + + +-- Version 1.1.20 --> 1.1.21: + +alter table camProcessedExp drop nastro; +alter table camProcessedExp add (fwhm double, fwhm_range double, n_stars int(11), n_extended int(11), n_cr int(11), n_astrom int(11)); +alter table chipProcessedImfile add column (bias double, bias_stdev double, fringe_0 double, fringe_1 double, fringe_2 double, sigma_ra double, sigma_dec double, ap_resid double, ap_resid_stdev double, fwhm double, fwhm_range double, n_stars int(11), n_extended int(11), n_cr int(11), n_astrom int(11)); +alter table warpSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; +alter table diffSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; +alter table stackSumSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; + + +-- Version 1.1.22 --> 1.1.23 +-- Adding support for reduction classes (which are used to specify recipes for parts of the pipeline). +# Adding 'reduction' to detRun +# Renaming 'recipe' in {chip,cam}{Pending,Processed}Exp to 'reduction' + +alter table detRun add column reduction varchar(64) NULL DEFAULT NULL after exp_type; +alter table chipPendingExp change column recipe reduction varchar(64); +alter table chipProcessedExp change column recipe reduction varchar(64); +alter table camPendingExp change column recipe reduction varchar(64); +alter table camProcessedExp change column recipe reduction varchar(64); + +-- Version ??? --> 1.1.29 + +alter table warpSkyfile add column ignored tinyint after good_frac; + +-- +-- The following set of changes were applied to update the gpc1 database on 2008/02/07 +-- + +alter table calDB change column catdir dvodb varchar(64); + +alter table camProcessedExp change column fwhm fwhm_major float; +alter table camProcessedExp change column fwhm_range fwhm_minor float; + +-- In order to keep the column order correct this is split up below +--alter table camProcessedExp add column ( bias float, bias_stdev float, fringe_0 float, fringe_1 float, fringe_2 float, ap_resid float, ap_resid_stdev float, dtime_detrend float, dtime_photom float, dtime_astrom float, hostname varchar(64) ); + +alter table camProcessedExp add column bias float after bg_mean_stdev; +alter table camProcessedExp add column bias_stdev float after bias; +alter table camProcessedExp add column fringe_0 float after bias_stdev; +alter table camProcessedExp add column fringe_1 float after fringe_0; +alter table camProcessedExp add column fringe_2 float after fringe_1; +alter table camProcessedExp add column ap_resid float after sigma_dec; +alter table camProcessedExp add column ap_resid_stdev float after ap_resid; +alter table camProcessedExp add column dtime_detrend float after fwhm_minor; +alter table camProcessedExp add column dtime_photom float after dtime_detrend; +alter table camProcessedExp add column dtime_astrom float after dtime_photom; +alter table camProcessedExp add column hostname varchar(64) after dtime_astrom; + + +alter table chipProcessedImfile change column fwhm fwhm_major float; +alter table chipProcessedImfile change column fwhm_range fwhm_minor float; + +-- In order to keep the column order correct this is split up below +-- alter table chipProcessedImfile add column ( zp_mean float, zp_stdev float, dtime_detrend float, dtime_photom float, dtime_astrom float, hostname varchar(64) ); + +alter table chipProcessedImfile add column zp_mean float after ap_resid_stdev; +alter table chipProcessedImfile add column zp_stdev float after zp_mean; +alter table chipProcessedImfile add column dtime_detrend float after fwhm_minor; +alter table chipProcessedImfile add column dtime_photom float after dtime_detrend; +alter table chipProcessedImfile add column dtime_astrom float after dtime_photom; +alter table chipProcessedImfile add column hostname varchar(64) after dtime_astrom; + + +-- In order to keep the column order correct this is split up below +-- alter table diffSkyfile add column ( dtime_diff float, hostname varchar(64) ); +alter table diffSkyfile add column dtime_diff float after bg_stdev; +alter table diffSkyfile add column hostname varchar(64) after dtime_diff; +alter table diffSkyfile change column good_frac good_frac float; + +alter table flatcorrRun add column filter varchar(64) after dvodb; + +alter table stackSumSkyfile change column good_frac good_frac float; + +-- alter table stackSumSkyfile add column ( dtime_stack float, hostname varchar(64) ); +alter table stackSumSkyfile add column dtime_stack float after bg_stdev; +alter table stackSumSkyfile add column hostname varchar(64) after dtime_stack; + +#alter table warpSkyfile add column ( dtime_warp float, hostname varchar(64) ); +alter table warpSkyfile add column dtime_warp float after bg_stdev; +alter table warpSkyfile add column hostname varchar(64) after dtime_warp; + +alter table warpSkyfile change column good_frac good_frac float; + +alter table newExp add column dvodb varchar(255) after reduction; +alter table newExp add column tess_id varchar(64) after dvodb; +alter table rawExp add column dvodb varchar(255) after reduction; +alter table rawExp add column tess_id varchar(64) after dvodb; +alter table chipRun add column tess_id varchar(64) after dvodb; +alter table camRun add column tess_id varchar(64) after dvodb; +alter table warpRun add column tess_id varchar(64) after dvodb; + +alter table camRun add column end_stage varchar(64) after tess_id; +alter table chipRun add column end_stage varchar(64) after tess_id; +alter table warpRun add column end_stage varchar(64) after tess_id; +alter table newExp add column end_stage varchar(64) after tess_id; +alter table rawExp add column end_stage varchar(64) after tess_id; +alter table camRun add key (end_stage); +alter table chipRun add key (end_stage); +alter table warpRun add key (end_stage); +alter table newExp add key (end_stage); +alter table rawExp add key (end_stage); + +alter table warpRun add column workdir_state varchar(64) after workdir; +alter table warpRun add key (workdir_state); + +alter table pzDoneImfile add column fault smallint(6); + +-- end of changes to 1.1.29 + +alter table newExp add column label varchar(64) after end_stage; + +-- 1.1.30 + +drop table pzDoneExp; +drop table pzDoneImfile; +drop table pzPendingExp; +drop table pzPendingImfile; +alter table flatcorrRun add column region VARCHAR(255) after stats; + +-- 1.1.31 + +ALTER TABLE warpRun ADD COLUMN cam_id BIGINT AFTER warp_id; +ALTER TABLE warpRun ADD KEY(cam_id); +ALTER TABLE warpRun ADD KEY(warp_id, cam_id); +ALTER TABLE warpRun ADD CONSTRAINT FOREIGN KEY(cam_id) REFERENCES +camRun(cam_id); +ALTER TABLE warpRun ADD COLUMN magiced TINYINT AFTER registered; +ALTER TABLE warpSkyCellMap DROP FOREIGN KEY warpSkyCellMap_ibfk_1; +UPDATE warpRun JOIN warpInputExp USING(warp_id) SET warpRun.cam_id = +warpInputExp.cam_id WHERE warpRun.warp_id = warpInputExp.warp_id; +ALTER TABLE warpSkyCellMap ADD CONSTRAINT FOREIGN KEY(warp_id, cam_id) +REFERENCES warpRun(warp_id, cam_id); +DROP TABLE warpInputExp; + +-- 1.1.32 +ALTER TABLE warpRun ADD COLUMN label VARCHAR(64) AFTER workdir_state; +ALTER TABLE warpRun ADD KEY(label); +CREATE TABLE warpMask (label VARCHAR(64), PRIMARY KEY(label)) ENGINE=innodb; + +-- 1.1.33 +ALTER TABLE summitExp ADD COLUMN fault smallint(6) NOT NULL AFTER imfiles; +ALTER TABLE summitExp ADD KEY(fault); + +-- 1.1.34 +ALTER TABLE rawImfile ADD KEY(exp_name); + +-- 1.1.35 +ALTER TABLE rawImfile ADD INDEX UNQIUE(exp_id, tmp_class_id); + +-- 1.1.36 +ALTER TABLE warpSkyfile ADD COLUMN (xmin INT, xmax INT, ymin INT, ymax INT); +ALTER TABLE diffSkyfile ADD COLUMN (stamps_num INT, stamps_rms FLOAT, sources INT); +ALTER TABLE rawImfile ADD COLUMN hostname VARCHAR(64) AFTER object; +ALTER TABLE rawExp ADD COLUMN hostname VARCHAR(64) AFTER object; + +-- 1.1.37 +ALTER TABLE pzDataStore ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER uri; +ALTER TABLE summitExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; +ALTER TABLE summitImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER uri; +ALTER TABLE pzDownloadExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER state; +ALTER TABLE pzDownloadImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; +ALTER TABLE newExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER label; +ALTER TABLE newImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER uri; +ALTER TABLE rawExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; +ALTER TABLE rawImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; + +-- 1.1.38 + +-- convert from utf8 -> latin1 for performance/space gains +SET FOREIGN_KEY_CHECKS=0; +ALTER TABLE calDB CONVERT TO CHARACTER SET latin1; +ALTER TABLE calRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE camMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE camProcessedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE camRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE detInputExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedStatImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detProcessedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRegisteredImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detResidExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detResidImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRunSummary CONVERT TO CHARACTER SET latin1; +ALTER TABLE detStackedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE flatcorrExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE flatcorrRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE guidePendingExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicNodeResult CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicSkyfileMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicTree CONVERT TO CHARACTER SET latin1; +ALTER TABLE newExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE newImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampDataStore CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampJob CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampRequest CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDataStore CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDownloadExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDownloadImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE rawExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE rawImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackSumSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE summitExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE summitImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpSkyCellMap CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpSkyfile CONVERT TO CHARACTER SET latin1; +SET FOREIGN_KEY_CHECKS=1; + +-- first use of fake* tables +-- drop and recreate any fake* tables created prior to this point + +DROP TABLE IF EXISTS fakeRun; +CREATE TABLE fakeRun ( + fake_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id), + KEY(cam_id), + KEY(state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(fake_id, cam_id), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +DROP TABLE IF EXISTS fakeProcessedImfile; +CREATE TABLE fakeProcessedImfile ( + fake_id BIGINT AUTO_INCREMENT, + exp_id BIGINT(20), + class_id VARCHAR(64), + uri VARCHAR(255), + dtime_fake FLOAT, + hostname VARCHAR(64), + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id), + FOREIGN KEY(exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +DROP TABLE IF EXISTS fakeMask; +CREATE TABLE fakeMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +-- re-normalize camProcessedExp table +-- unrelated to other changes +ALTER TABLE camProcessedExp DROP FOREIGN KEY camProcessedExp_ibfk_1; +ALTER TABLE camProcessedExp DROP COLUMN chip_id; +ALTER TABLE camProcessedExp ADD FOREIGN KEY (cam_id) REFERENCES camRun (cam_id); + +-- insert fakeRun betwen camRun and warpRun +-- populate fakeRun/fakeProcessedImfile +-- then heal the foreign key references + +CREATE TEMPORARY TABLE warpCamMap ( + warp_id BIGINT, + cam_id BIGINT, + PRIMARY KEY(warp_id), + KEY(cam_id) +) ENGINE=MEMORY; + +INSERT INTO warpCamMap SELECT warp_id, cam_id FROM warpRun; + +ALTER TABLE warpRun drop FOREIGN KEY warpRun_ibfk_1; +ALTER TABLE warpSkyCellMap DROP FOREIGN KEY warpSkyCellMap_ibfk_1; + +ALTER TABLE warpRun change COLUMN cam_id fake_id BIGINT(20) DEFAULT NULL; +ALTER TABLE warpSkyCellMap DROP COLUMN cam_id; + +SET FOREIGN_KEY_CHECKS=0; +ALTER TABLE warpRun ADD FOREIGN KEY (fake_id) REFERENCES fakeRun (fake_id); +SET FOREIGN_KEY_CHECKS=1; +ALTER TABLE warpSkyCellMap ADD FOREIGN KEY (warp_id) REFERENCES warpRun(warp_id); + +INSERT INTO fakeRun +SELECT + NULL, + camRun.cam_id, + 'stop', + workdir, + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage, + NULL +FROM camRun +JOIN warpCamMap + using(cam_id); + +INSERT INTO fakeProcessedImfile +SELECT + fake_id, + chipRun.exp_id, + class_id, + uri, + 0, + hostname, + path_base, + fault, + NULL +FROM fakeRun +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id); + +UPDATE warpRun, fakeRun, warpCamMap +SET warpRun.fake_id = fakeRun.fake_id +WHERE warpRun.warp_id = warpCamMap.warp_id + AND warpCamMap.cam_id = fakeRun.fake_id; + +DROP TABLE warpCamMap; + + +-- new values for chipRun.state + +update chipRun set state = 'new' where state = 'run'; +update chipRun set state = 'full' where state = 'stop'; + +-- from eam_branch_20080806 +-- add 'data_state' to detrend tables + +alter table detProcessedImfile add column data_state varchar(64) after path_base; +alter table detProcessedExp add column data_state varchar(64) after path_base; +alter table detStackedImfile add column data_state varchar(64) after user_5; +alter table detNormalizedStatImfile add column data_state varchar(64) after norm; +alter table detNormalizedImfile add column data_state varchar(64) after path_base; +alter table detNormalizedExp add column data_state varchar(64) after path_base; +alter table detResidImfile add column data_state varchar(64) after path_base; +alter table detResidExp add column data_state varchar(64) after path_base; +alter table detRunSummary add column data_state varchar(64) after iteration; +alter table detRegisteredImfile add column data_state varchar(64) after path_base; + +update camRun set state = 'new' where state = 'run'; +update camRun set state = 'full' where state = 'stop'; + +update fakeRun set state = 'new' where state = 'run'; +update fakeRun set state = 'full' where state = 'stop'; + +update warpRun set state = 'new' where state = 'run'; +update warpRun set state = 'full' where state = 'stop'; + +update stackRun set state = 'new' where state = 'run'; +update stackRun set state = 'full' where state = 'stop'; + +update diffRun set state = 'new' where state = 'run'; +update diffRun set state = 'full' where state = 'stop'; + +-- not sure when 'filter' was added to stackRun, but it is needed now: + +alter table stackRun add column filter varchar(64) after tess_id; + +-- changes to the flatcorr tables : no valid flatcorr tables have been created to date expect my tests +-- this sql just drops and recreates the flatcorrRun with the correct layout + +delete from flatcorrExp; +delete from flatcorrRun; + +drop table flatcorrExp; +drop table flatcorrRun; + +CREATE TABLE flatcorrRun ( + corr_id BIGINT AUTO_INCREMENT, + dvodb VARCHAR(64), + filter VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + region VARCHAR(64), + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(corr_id), + KEY(corr_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrChipLink ( + corr_id BIGINT, + chip_id BIGINT, + PRIMARY KEY(corr_id, chip_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrCamLink ( + corr_id BIGINT, + chip_id BIGINT, + cam_id BIGINT, + PRIMARY KEY(corr_id, chip_id, cam_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id), + FOREIGN KEY (cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- I initially defined flatcorrCamLin without the chip_id entry. this +-- command corrects for that, but should not be needed widely. +-- ALTER TABLE flatcorrCamLink ADD COLUMN chip_id BIGINT AFTER corr_id; +-- alter table flatcorrRun add column hostname varchar(64) after region; +-- alter table flatcorrRun add column fault smallint after hostname; +-- alter table flatcorrRun add column reduction varchar(64) after stats; + +-- changes to the chip tables to support cleanup and reprocessing + +ALTER TABLE chipProcessedImfile ADD COLUMN data_state VARCHAR(64) AFTER class_id; +UPDATE chipProcessedImfile SET data_state = 'full'; + +-- I have added 'label' and 'reduction' entries to stack and diff + +ALTER TABLE stackRun ADD COLUMN label VARCHAR(64) AFTER workdir; +ALTER TABLE stackRun ADD COLUMN reduction VARCHAR(64) AFTER label; + +ALTER TABLE stackSumSkyfile add column data_state varchar(64) after path_base; +UPDATE stackSumSkyfile SET data_state = 'full'; + +alter table diffRun add column label varchar(64) after workdir; +alter table diffRun add column reduction varchar(64) after label; + +ALTER TABLE diffSkyfile add column data_state varchar(64) after path_base; +UPDATE diffSkyfile SET data_state = 'full'; + +ALTER TABLE warpSkyfile add column data_state varchar(64) after path_base; +UPDATE warpSkyfile SET data_state = 'full'; + +ALTER TABLE fakeProcessedImfile add column data_state varchar(64) after path_base; +update fakeProcessedImfile set data_state ='full'; + +-- adding ref_det_id and ref_iter to detRun and detResidImfile + +ALTER TABLE detResidImfile add column ref_det_id bigint after iteration; +ALTER TABLE detResidImfile add column ref_iter int after ref_det_id; + +ALTER TABLE detRun change column parent ref_det_id bigint; +ALTER TABLE detRun add column ref_iter int after ref_det_id; + +-- populate ref_det_id and ref_iter +UPDATE detResidImfile SET ref_det_id = det_id, ref_iter = iteration; + +-- Next we drop the old constraint and add the new one +ALTER TABLE detResidImfile DROP FOREIGN KEY detResidImfile_ibfk_3; +ALTER TABLE detResidImfile add FOREIGN KEY (ref_det_id, ref_iter) + REFERENCES detNormalizedExp(det_id, iteration); + +--- Additions for stack/diff QA by PAP. +ALTER TABLE diffSkyfile ADD COLUMN stamps_mean FLOAT AFTER stamps_num; +ALTER TABLE diffSkyfile ADD COLUMN dtime_match FLOAT AFTER dtime_diff; +ALTER TABLE diffSkyfile ADD COLUMN dtime_phot FLOAT AFTER dtime_match; + +ALTER TABLE stackSumSkyfile ADD COLUMN dtime_match_mean FLOAT AFTER dtime_stack; +ALTER TABLE stackSumSkyfile ADD COLUMN dtime_match_stdev FLOAT AFTER dtime_match_mean; +ALTER TABLE stackSumSkyfile ADD COLUMN dtime_initial FLOAT AFTER dtime_match_stdev; +ALTER TABLE stackSumSkyfile ADD COLUMN dtime_reject FLOAT AFTER dtime_initial; +ALTER TABLE stackSumSkyfile ADD COLUMN dtime_final FLOAT AFTER dtime_reject; +ALTER TABLE stackSumSkyfile ADD COLUMN dtime_phot FLOAT AFTER dtime_final; +ALTER TABLE stackSumSkyfile ADD COLUMN match_mean FLOAT AFTER dtime_phot; +ALTER TABLE stackSumSkyfile ADD COLUMN match_stdev FLOAT AFTER match_mean; +ALTER TABLE stackSumSkyfile ADD COLUMN match_rms FLOAT AFTER match_stdev; +ALTER TABLE stackSumSkyfile ADD COLUMN stamps_mean FLOAT AFTER match_rms; +ALTER TABLE stackSumSkyfile ADD COLUMN stamps_stdev FLOAT AFTER stamps_mean; +ALTER TABLE stackSumSkyfile ADD COLUMN stamps_min INT AFTER stamps_stdev; +ALTER TABLE stackSumSkyfile ADD COLUMN reject_images INT AFTER stamps_min; +ALTER TABLE stackSumSkyfile ADD COLUMN reject_pix_mean FLOAT AFTER reject_images; +ALTER TABLE stackSumSkyfile ADD COLUMN reject_pix_stdev FLOAT AFTER reject_pix_mean; +ALTER TABLE stackSumSkyfile ADD COLUMN sources INT AFTER reject_pix_stdev; +ALTER TABLE stackSumSkyfile DROP COLUMN data_state; +ALTER TABLE diffSkyfile DROP COLUMN data_state; + +-- Adding the new solar-system info (sun & moon data) +ALTER TABLE rawExp CHANGE COLUMN solang sun_angle FLOAT; +ALTER TABLE rawExp ADD COLUMN sun_alt FLOAT AFTER sun_angle; +ALTER TABLE rawExp ADD COLUMN moon_angle FLOAT AFTER sun_alt; +ALTER TABLE rawExp ADD COLUMN moon_alt FLOAT AFTER moon_angle; +ALTER TABLE rawExp ADD COLUMN moon_phase FLOAT AFTER moon_alt; + +ALTER TABLE rawImfile ADD COLUMN sun_angle FLOAT AFTER object; +ALTER TABLE rawImfile ADD COLUMN sun_alt FLOAT AFTER sun_angle; +ALTER TABLE rawImfile ADD COLUMN moon_angle FLOAT AFTER sun_alt; +ALTER TABLE rawImfile ADD COLUMN moon_alt FLOAT AFTER moon_angle; +ALTER TABLE rawImfile ADD COLUMN moon_phase FLOAT AFTER moon_alt; + +CREATE TABLE pstampProject ( + proj_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + dbname VARCHAR(64), + dvodb VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + need_magic TINYINT, + PRIMARY KEY(proj_id), + KEY(proj_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +-- Reusing astrometry time field since ppImage doesn't do that any more +ALTER TABLE chipProcessedImfile CHANGE COLUMN dtime_astrom dtime_total FLOAT; + +-- Deleting fields requiring astrometric matching from chipProcessedImfile +ALTER TABLE chipProcessedImfile DROP COLUMN sigma_ra; +ALTER TABLE chipProcessedImfile DROP COLUMN sigma_dec; +ALTER TABLE chipProcessedImfile DROP COLUMN zp_mean; +ALTER TABLE chipProcessedImfile DROP COLUMN zp_stdev; +ALTER TABLE chipProcessedImfile DROP COLUMN n_astrom; + +-- No need to propagate processing times from chip to camera +ALTER TABLE camProcessedExp DROP COLUMN dtime_detrend; +ALTER TABLE camProcessedExp DROP COLUMN dtime_photom; +ALTER TABLE camProcessedExp DROP COLUMN dtime_astrom; + +-- Add "dtime_script" to processing results +ALTER TABLE chipProcessedImfile ADD COLUMN dtime_script FLOAT AFTER dtime_total; +ALTER TABLE camProcessedExp ADD COLUMN dtime_script FLOAT AFTER fwhm_minor; +ALTER TABLE fakeProcessedImfile ADD COLUMN dtime_script FLOAT AFTER dtime_fake; +ALTER TABLE warpSkyfile ADD COLUMN dtime_script FLOAT AFTER dtime_warp; +ALTER TABLE stackSumSkyfile ADD COLUMN dtime_script FLOAT AFTER dtime_phot; +ALTER TABLE diffSkyfile ADD COLUMN dtime_script FLOAT AFTER dtime_phot; + +* 1.1.40 + +-- Add moments statistics (M2, M3, M4) + +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2 FLOAT AFTER fwhm_minor; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2_err FLOAT AFTER iq_m2; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2_lq FLOAT AFTER iq_m2_err; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2_uq FLOAT AFTER iq_m2_lq; + +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2c FLOAT AFTER iq_m2_uq; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2c_err FLOAT AFTER iq_m2c; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2c_lq FLOAT AFTER iq_m2c_err; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2c_uq FLOAT AFTER iq_m2c_lq; + +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2s FLOAT AFTER iq_m2c_uq; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2s_err FLOAT AFTER iq_m2s; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2s_lq FLOAT AFTER iq_m2s_err; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m2s_uq FLOAT AFTER iq_m2s_lq; + +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m3 FLOAT AFTER iq_m2s_uq; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m3_err FLOAT AFTER iq_m3; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m3_lq FLOAT AFTER iq_m3_err; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m3_uq FLOAT AFTER iq_m3_lq; + +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m4 FLOAT AFTER iq_m3_uq; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m4_err FLOAT AFTER iq_m4; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m4_lq FLOAT AFTER iq_m4_err; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_m4_uq FLOAT AFTER iq_m4_lq; + +ALTER TABLE chipProcessedImfile ADD COLUMN n_psfstars INT AFTER n_stars; +ALTER TABLE chipProcessedImfile ADD COLUMN n_iqstars INT AFTER n_psfstars; + + +ALTER TABLE camProcessedExp ADD COLUMN iq_m2 FLOAT AFTER fwhm_minor; +ALTER TABLE camProcessedExp ADD COLUMN iq_m2_err FLOAT AFTER iq_m2; +ALTER TABLE camProcessedExp ADD COLUMN iq_m2_lq FLOAT AFTER iq_m2_err; +ALTER TABLE camProcessedExp ADD COLUMN iq_m2_uq FLOAT AFTER iq_m2_lq; + +ALTER TABLE camProcessedExp ADD COLUMN iq_m2c FLOAT AFTER iq_m2_uq; +ALTER TABLE camProcessedExp ADD COLUMN iq_m2c_err FLOAT AFTER iq_m2c; +ALTER TABLE camProcessedExp ADD COLUMN iq_m2c_lq FLOAT AFTER iq_m2c_err; +ALTER TABLE camProcessedExp ADD COLUMN iq_m2c_uq FLOAT AFTER iq_m2c_lq; + +ALTER TABLE camProcessedExp ADD COLUMN iq_m2s FLOAT AFTER iq_m2c_uq; +ALTER TABLE camProcessedExp ADD COLUMN iq_m2s_err FLOAT AFTER iq_m2s; +ALTER TABLE camProcessedExp ADD COLUMN iq_m2s_lq FLOAT AFTER iq_m2s_err; +ALTER TABLE camProcessedExp ADD COLUMN iq_m2s_uq FLOAT AFTER iq_m2s_lq; + +ALTER TABLE camProcessedExp ADD COLUMN iq_m3 FLOAT AFTER iq_m2s_uq; +ALTER TABLE camProcessedExp ADD COLUMN iq_m3_err FLOAT AFTER iq_m3; +ALTER TABLE camProcessedExp ADD COLUMN iq_m3_lq FLOAT AFTER iq_m3_err; +ALTER TABLE camProcessedExp ADD COLUMN iq_m3_uq FLOAT AFTER iq_m3_lq; + +ALTER TABLE camProcessedExp ADD COLUMN iq_m4 FLOAT AFTER iq_m3_uq; +ALTER TABLE camProcessedExp ADD COLUMN iq_m4_err FLOAT AFTER iq_m4; +ALTER TABLE camProcessedExp ADD COLUMN iq_m4_lq FLOAT AFTER iq_m4_err; +ALTER TABLE camProcessedExp ADD COLUMN iq_m4_uq FLOAT AFTER iq_m4_lq; + +ALTER TABLE camProcessedExp ADD COLUMN n_psfstars INT AFTER n_stars; +ALTER TABLE camProcessedExp ADD COLUMN n_iqstars INT AFTER n_psfstars; + +* 1.1.41 + +ALTER TABLE camProcessedExp ADD COLUMN dtime_astrom FLOAT AFTER dtime_script; +ALTER TABLE camProcessedExp ADD COLUMN dtime_addstar FLOAT AFTER dtime_astrom; + +* 1.1.42 + +ALTER TABLE diffSkyfile ADD COLUMN norm FLOAT AFTER stamps_rms; +ALTER TABLE diffSkyfile ADD COLUMN bg_diff FLOAT AFTER norm; +ALTER TABLE diffSkyfile ADD COLUMN kernel_x FLOAT AFTER bg_diff; +ALTER TABLE diffSkyfile ADD COLUMN kernel_y FLOAT AFTER kernel_x; +ALTER TABLE diffSkyfile ADD COLUMN kernel_xx FLOAT AFTER kernel_y; +ALTER TABLE diffSkyfile ADD COLUMN kernel_xy FLOAT AFTER kernel_xx; +ALTER TABLE diffSkyfile ADD COLUMN kernel_yy FLOAT AFTER kernel_xy; + +* 1.1.43 + +ALTER TABLE rawExp ADD COLUMN obs_mode VARCHAR(64) AFTER comment; +ALTER TABLE rawExp ADD COLUMN obs_group VARCHAR(64) AFTER obs_mode; + +ALTER TABLE rawImfile ADD COLUMN obs_mode VARCHAR(64) AFTER comment; +ALTER TABLE rawImfile ADD COLUMN obs_group VARCHAR(64) AFTER obs_mode; + +* 1.1.44 + +ALTER TABLE chipProcessedImfile ADD COLUMN fwhm_major_lq FLOAT AFTER fwhm_major; +ALTER TABLE chipProcessedImfile ADD COLUMN fwhm_major_uq FLOAT AFTER fwhm_major_lq; + +ALTER TABLE chipProcessedImfile ADD COLUMN fwhm_minor_lq FLOAT AFTER fwhm_minor; +ALTER TABLE chipProcessedImfile ADD COLUMN fwhm_minor_uq FLOAT AFTER fwhm_minor_lq; + +ALTER TABLE chipProcessedImfile ADD COLUMN iq_fwhm_major FLOAT AFTER fwhm_minor_uq; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_fwhm_major_err FLOAT AFTER iq_fwhm_major; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_fwhm_minor FLOAT AFTER iq_fwhm_major_err; +ALTER TABLE chipProcessedImfile ADD COLUMN iq_fwhm_minor_err FLOAT AFTER iq_fwhm_minor; + +ALTER TABLE camProcessedExp ADD COLUMN fwhm_major_lq FLOAT AFTER fwhm_major; +ALTER TABLE camProcessedExp ADD COLUMN fwhm_major_uq FLOAT AFTER fwhm_major_lq; + +ALTER TABLE camProcessedExp ADD COLUMN fwhm_minor_lq FLOAT AFTER fwhm_minor; +ALTER TABLE camProcessedExp ADD COLUMN fwhm_minor_uq FLOAT AFTER fwhm_minor_lq; + +ALTER TABLE camProcessedExp ADD COLUMN iq_fwhm_major FLOAT AFTER fwhm_minor_uq; +ALTER TABLE camProcessedExp ADD COLUMN iq_fwhm_major_err FLOAT AFTER iq_fwhm_major; +ALTER TABLE camProcessedExp ADD COLUMN iq_fwhm_minor FLOAT AFTER iq_fwhm_major_err; +ALTER TABLE camProcessedExp ADD COLUMN iq_fwhm_minor_err FLOAT AFTER iq_fwhm_minor; + +ALTER TABLE flatcorrRun ADD COLUMN camera VARCHAR(64) AFTER dvodb; +ALTER TABLE flatcorrRun ADD COLUMN telescope VARCHAR(64) AFTER camera; +ALTER TABLE flatcorrRun ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER telescope; + +* 1.1.45 + +ALTER TABLE camProcessedExp CHANGE COLUMN zp_mean zpt_obs FLOAT; +ALTER TABLE camProcessedExp CHANGE COLUMN zp_stdev zpt_stdev FLOAT; +ALTER TABLE camProcessedExp ADD COLUMN zpt_lq FLOAT AFTER zpt_stdev; +ALTER TABLE camProcessedExp ADD COLUMN zpt_uq FLOAT AFTER zpt_lq; + +* 1.1.46 + +ALTER TABLE pzDownloadImfile ADD COLUMN hostname varchar(64) after epoch; +ALTER TABLE flatcorrRun ADD COLUMN det_type varchar(64) after corr_id; +ALTER TABLE flatcorrChipLink ADD COLUMN include tinyint after chip_id; +ALTER TABLE flatcorrCamLink ADD COLUMN include tinyint after cam_id; + +ALTER TABLE rawImfile ADD COLUMN ignored TINYINT AFTER moon_phase; + Index: /branches/pap_branches/pap_branch_20081109/dbconfig/chip.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/chip.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/chip.md (revision 22235) @@ -0,0 +1,87 @@ +chipRun METADATA + chip_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 64 # Key INDEX(chip_id, exp_id) fkey (exp_id) ref rawExp(exp_id) + state STR 64 # Key + workdir STR 255 + workdir_state STR 64 # Key + label STR 64 # Key + reduction STR 64 # Reduction class + expgroup STR 64 # Key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key +END + +chipProcessedImfile METADATA + chip_id S64 0 # Primary Key fkey (chip_id, exp_id) ref chipRun(chip_id, exp_id) + exp_id S64 64 # Primary Key fkey (exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key + data_state STR 64 # Key + uri STR 255 + bg F32 0.0 + bg_stdev F32 0.0 + bg_mean_stdev F32 0.0 + bias F32 0.0 + bias_stdev F32 0.0 + fringe_0 F32 0.0 + fringe_1 F32 0.0 + fringe_2 F32 0.0 + ap_resid F32 0.0 + ap_resid_stdev F32 0.0 + + fwhm_major F32 0.0 + fwhm_major_lq F32 0.0 + fwhm_major_uq F32 0.0 + + fwhm_minor F32 0.0 + fwhm_minor_lq F32 0.0 + fwhm_minor_uq F32 0.0 + + iq_fwhm_major F32 0.0 + iq_fwhm_major_err F32 0.0 + iq_fwhm_minor F32 0.0 + iq_fwhm_minor_err F32 0.0 + + iq_m2 F32 0.0 + iq_m2_err F32 0.0 + iq_m2_lq F32 0.0 + iq_m2_uq F32 0.0 + + iq_m2c F32 0.0 + iq_m2c_err F32 0.0 + iq_m2c_lq F32 0.0 + iq_m2c_uq F32 0.0 + + iq_m2s F32 0.0 + iq_m2s_err F32 0.0 + iq_m2s_lq F32 0.0 + iq_m2s_uq F32 0.0 + + iq_m3 F32 0.0 + iq_m3_err F32 0.0 + iq_m3_lq F32 0.0 + iq_m3_uq F32 0.0 + + iq_m4 F32 0.0 + iq_m4_err F32 0.0 + iq_m4_lq F32 0.0 + iq_m4_uq F32 0.0 + + dtime_detrend F32 0.0 + dtime_photom F32 0.0 + dtime_total F32 0.0 + dtime_script F32 0.0 + + hostname STR 64 + n_stars S32 0 + n_psfstars S32 0 + n_iqstars S32 0 + n_extended S32 0 + n_cr S32 0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +chipMask METADATA + label STR 64 # Primary Key +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/config.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/config.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/config.md (revision 22235) @@ -0,0 +1,5 @@ +glueforge METADATA + pkg_name STR ippdb + pkg_namespace STR ippdb + pkg_version STR 1.1.46 +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/det.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/det.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/det.md (revision 22235) @@ -0,0 +1,276 @@ +detRun METADATA + det_id S64 0 # Primary Key AUTO_INCREMENT + iteration S32 0 # Key INDEX(det_id, iteration) + det_type STR 64 # Key + mode STR 64 # Key + state STR 64 # Key + filelevel STR 64 + workdir STR 255 # destination for output files + camera STR 64 + telescope STR 64 + exp_type STR 64 # XXX this should be dropped + reduction STR 64 # Reduction clas + filter STR 64 + airmass_min F32 0.0 + airmass_max F32 0.0 + exp_time_min F32 0.0 + exp_time_max F32 0.0 + ccd_temp_min F32 0.0 + ccd_temp_max F32 0.0 + posang_min F64 0.0 + posang_max F64 0.0 + registered TAI 0001-01-01T00:00:00Z + time_begin TAI 0001-01-01T00:00:00Z + time_end TAI 0001-01-01T00:00:00Z + use_begin TAI 0001-01-01T00:00:00Z + use_end TAI 0001-01-01T00:00:00Z + solang_min F32 0.0 + solang_max F32 0.0 + label STR 64 # key + ref_det_id S64 0 # reference for 'verify' and 'correction' analysis + ref_iter S32 0 # reference for 'verify' and 'correction' analysis +END + +detInputExp METADATA + det_id S64 0 # Primary Key fkey(det_id) ref detRun(det_id) + iteration S32 0 # Primary Key fkey(exp_id) ref rawExp(exp_id) + exp_id S64 64 # Primary Key INDEX(det_id, exp_id) + include BOOL f # INDEX(det_id, iteration) +END + +detProcessedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key INDEX(det_id, class_id) + uri STR 255 # INDEX(det_id, exp_id) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detProcessedExp METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(det_id, exp_id) ref detProcessedImfile(det_id, exp_id) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +# +# detStackedImfile does not depend on detProcessedExp. detProcesedExp is purely +# FYI values and can be calculated in parellel +# +detStackedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, class_id) ref detProcessedImfile(det_id, class_id) + class_id STR 64 # Primary Key + uri STR 255 + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 +# XXX does it make sense to 'clean' the stacked imfiled? + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detNormalizedStatImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, iteration, class_id) ref detStackedImfile(det_id, iteration, class_id) + class_id STR 64 # Primary Key + norm F32 0.0 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +## XXX this must match in fields with detRegisteredImfile +detNormalizedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id) ref detInputExp(det_id) + iteration S32 0 # Primary Key fkey(det_id, iteration, class_id) ref detNormalizedStatImfile(det_id, iteration, class_id) + class_id STR 64 # Primary Key INDEX(det_id, iteration) + uri STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detNormalizedExp METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, iteration) ref detNormalizedImfile(det_id, iteration) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +#detMasterFrame METADATA +# det_id S64 0 # Primary Key +# iteration S32 0 # Primary Key +# comment STR 255 +#END +# +## drop? +#detMasterImfile METADATA +# det_id S64 0 # Primary Key +# class_id STR 64 # Primary Key +# uri STR 255 +# recipe STR 64 +#END + +detResidImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration, exp_id) ref detInputExp(det_id, iteration, exp_id) + iteration S32 0 # Primary Key fkey(det_id, exp_id, class_id) ref detProcessedImfile(det_id, exp_id, class_id) + ref_det_id S64 0 # detrend master actually applied (same as above for 'master', but not for 'verify') + ref_iter S32 0 # detrend master actually applied (same as above for 'master', but not for 'verify') + exp_id S64 64 # Primary Key fkey(det_id, iteration) ref detNormalizedExp(det_id, iteration) + class_id STR 64 # Primary Key INDEX(det_id, iteration, exp_id) + uri STR 255 + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + bg_skewness F64 0.0 + bg_kurtosis F64 0.0 + bin_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + fringe_resid_0 F64 0.0 + fringe_resid_1 F64 0.0 + fringe_resid_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detResidExp METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration, exp_id) ref detInputExp(det_id, iteration, exp_id) + iteration S32 0 # Primary Key fkey(det_id, iteration, exp_id) ref detResidImfile(det_id, iteration, exp_id) + exp_id S64 64 # Primary Key INDEX(det_id, iteration) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + bg_skewness F64 0.0 + bg_kurtosis F64 0.0 + bin_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + fringe_resid_0 F64 0.0 + fringe_resid_1 F64 0.0 + fringe_resid_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + accept BOOL f + fault S16 0 # Key NOT NULL +END + +# XXX this probably should have been defined at the start of a new iteration, not at the end. +detRunSummary METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, iteration) ref detResidExp(det_id, iteration) + data_state STR 64 # full, goto_cleaned, cleaned, goto_full, goto_purged, purged + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + accept BOOL f + fault S16 0 # Key NOT NULL +END + +# +# Note: This table needs to stay more or less identical to detNormalizedImfile. +# It only exists as a seperate entity so they it can have different fkeys +# +detRegisteredImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detRun(det_id, iteration) + iteration S32 0 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detCorrectedExp METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(exp_id, class_id) ref rawImfile(exp_id, class_id) + uri STR 255 # INDEX(det_id, exp_id) + corr_id S64 0 + corr_type STR 64 + recipe STR 64 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +detCorrectedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key INDEX(det_id, class_id) + uri STR 255 # INDEX(det_id, exp_id) + path_base STR 255 + fault S16 0 # Key NOT NULL +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/diff.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/diff.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/diff.md (revision 22235) @@ -0,0 +1,55 @@ +# $Id: diff.md,v 1.14 2008-10-11 02:33:40 price Exp $ + +diffRun METADATA + diff_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 # Key + label STR 64 # Key + reduction STR 64 # Reduction class + workdir STR 255 + dvodb STR 255 + registered TAI NULL + skycell_id STR 64 # Key + tess_id STR 64 # Key +END + +# +# Diff Sky Cells Mode +# + +# only ever 2 per run - one template / one not +diffInputSkyfile METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + template BOOL f # Primary Key + stack_id S64 0 # fkey(stack_id) ref stackSumSkyfile(stack_id) + warp_id S64 0 # fkey(warp_id, skycell_id, tess_id) ref warpSkyfile(warp_id, skycell_id, tess_id) + skycell_id STR 64 # Key + tess_id STR 64 # Key +# either a input or a template + kind STR 64 # Key +END + +diffSkyfile METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + uri STR 255 + path_base STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + stamps_num S32 0 + stamps_mean F32 0.0 + stamps_rms F32 0.0 + norm F32 0.0 + bg_diff F32 0.0 + kernel_x F32 0.0 + kernel_y F32 0.0 + kernel_xx F32 0.0 + kernel_xy F32 0.0 + kernel_yy F32 0.0 + sources S32 0 + dtime_diff F32 0.0 + dtime_match F32 0.0 + dtime_phot F32 0.0 + dtime_script F32 0.0 + hostname STR 64 + good_frac F32 0.0 # Key + fault S16 0 # Key +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/dimm.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/dimm.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/dimm.md (revision 22235) @@ -0,0 +1,21 @@ +# Table 13: DIMM Measurements Table +# Column Name Datatype Description +# Time date/time The time the DIMM observation was taken. +# sigmax float Raw dispersion in x. +# sigmay float Raw dispersion in y. +# FWHM float Dervied seeing full width at half maximum. +# RA float The coordinates of the measured star. +# DEC float The coordinates of the measured star. +# Exposure time float The exposure time of the DIMM observation. +# Telescope ID string source of the DIMM data + +dimm METADATA +# time DATETIME 2006-01-11T00:00:00 + sigmax F32 0.0 + sigmay F32 0.0 + fwhm F32 0.0 + ra F64 0.0 + decl F64 0.0 + expttime F32 0.0 + telescope_id STR 255 +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/dome.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/dome.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/dome.md (revision 22235) @@ -0,0 +1,16 @@ +# Table 15: Dome Status Table +# Column Name Datatype Description +# Time date/time The time for which the dome status is valid. +# Azimuth float The azimuth of the dome. +# Open status boolean Whether the dome is open or not. +# Lights status boolean Whether lights are on in the dome or not. +# Track status boolean Whether dome is tracking telescope or not. + +dome METADATA +# time DATETIME 2006-01-11T00:00:00 + az F32 0.0 + open BOOL t + light BOOL t + # XXX is it possible for the dome slit to not track the telescope? ;) + track BOOL t +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/fake.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/fake.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/fake.md (revision 22235) @@ -0,0 +1,31 @@ +fakeRun METADATA + fake_id S64 0 # Primary Key AUTO_INCREMENT + cam_id S64 0 # Key INDEX(fake_id, cam_id) fkey (cam_id) ref camRun(cam_id) + state STR 64 # Key + workdir STR 255 + label STR 64 # Key + reduction STR 64 # Reduction class + expgroup STR 64 # Key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +fakeProcessedImfile METADATA + fake_id S64 0 # Primary Key fkey (fake_id) ref fakeRun(fake_id) + exp_id S64 64 # Primary Key fkey (exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key + uri STR 255 + dtime_fake F32 0.0 + dtime_script F32 0.0 + hostname STR 64 + path_base STR 255 + data_state STR 64 # Key + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +fakeMask METADATA + label STR 64 # Primary Key +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/flatcorr.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/flatcorr.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/flatcorr.md (revision 22235) @@ -0,0 +1,34 @@ + +# table of flat-field correction runs +flatcorrRun METADATA + corr_id S64 0 # Primary Key AUTO_INCREMENT + det_type STR 64 # type of output detrend correction (eg, FLATCORR, FLATTEST) + dvodb STR 64 + camera STR 64 + telescope STR 64 + epoch UTC 0001-01-01T00:00:00Z + filter STR 64 + state STR 64 + workdir STR 255 + label STR 64 + reduction STR 64 + region STR 64 + hostname STR 64 + fault S16 0 # Key NOT NULL +END + +# table of Exposure-level data used for each flat-field corrction run +flatcorrChipLink METADATA + corr_id S64 0 # Primary Key fkey(corr_id) ref flatcorrRun(corr_id) + chip_id S64 0 # Primary Key fkey(chip_id) ref chipRun(chip_id) + include BOOL t +END + +# table of Exposure-level data used for each flat-field corrction run +flatcorrCamLink METADATA + corr_id S64 0 # Primary Key fkey(corr_id) ref flatcorrRun(corr_id) + chip_id S64 0 # Primary Key fkey(chip_id) ref chipRun(chip_id) + cam_id S64 0 # Primary Key fkey(chip_id) ref chipRun(chip_id) + include BOOL t +END + Index: /branches/pap_branches/pap_branch_20081109/dbconfig/guide.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/guide.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/guide.md (revision 22235) @@ -0,0 +1,5 @@ +guidePendingExp METADATA + guide_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 64 # Key + recipe STR 64 +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/ipp.m4 =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/ipp.m4 (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/ipp.m4 (revision 22235) @@ -0,0 +1,23 @@ +include(config.md) +dnl include(weather.md) +dnl include(skyp_transparency.md) +dnl include(skyp_absorption.md) +dnl include(skyp_emission.md) +dnl include(dimm.md) +dnl include(skyp_ir.md) +dnl include(dome.md) +dnl include(telescope.md) +dnl include(skycell.md) +include(tasks.md) +include(guide.md) +include(chip.md) +include(cam.md) +include(fake.md) +include(warp.md) +include(diff.md) +include(stack.md) +include(det.md) +include(magic.md) +include(calibration.md) +include(flatcorr.md) +include(pstamp.md) Index: /branches/pap_branches/pap_branch_20081109/dbconfig/magic.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/magic.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/magic.md (revision 22235) @@ -0,0 +1,47 @@ +# $Id: magic.md,v 1.10 2008-07-23 01:52:30 price Exp $ + +### Fault in magicRun indicates that the processing tree failed +magicRun METADATA + magic_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 0 # Key + state STR 64 # Key + workdir STR 255 + workdir_state STR 255 # Key + label STR 64 # key + dvodb STR 255 + registered TAI NULL + fault S16 0 # Key +END + +magicInputSkyfile METADATA + magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + node STR 64 # +END + +magicTree METADATA + magic_id S64 0 # Key fkey(magic_id) ref magicRun(magic_id) + node STR 64 # Key INDEX(magic_id, node) + dep STR 64 # Key +END + +magicNodeResult METADATA + magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + node STR 64 # Primary Key fkey(magic_id, node) ref magicTree(magic_id, node) + uri STR 255 + fault S16 0 # Key +END + +magicMask METADATA + magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + uri STR 255 + streaks S32 0 + fault S16 0 # Key +END + +### I don't think we need this if magic produces mask descriptions. -- PAP. +#magicSkyfileMask METADATA +# magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) +# diff_id S64 0 # Primary Key fkey(magic_id, diff_id) ref magicInputSkyfile(magic_id, diff_id) +# uri STR 255 # fkey(magic_id) ref magicMask(magic_id) +#END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/notes.txt =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/notes.txt (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/notes.txt (revision 22235) @@ -0,0 +1,41 @@ + +2007.06.08 : EAM + + * when changing the database schema: + * increment the pkg_version number on dbconfig/config.md + * increment the ippdb version number in ippTools/configure.ac (to match) + * increment the ippTools version number in ippTools/configure.ac + * build ippbd ('make src' in dbconfig) + * check in dbconfig, ippdb, and ippTools + +2007.06.04 : EAM + +I am adding additional fields which we will need for selecting the +input detrend images: + +rawExp : + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + +detProcessedImfile, detProcessedExp: + fringe amplitude + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + +detResidImfile, detResidExp: + binned stdev + fringe_0 + fringe_1 + fringe_2 + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + Index: /branches/pap_branches/pap_branch_20081109/dbconfig/pstamp.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/pstamp.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/pstamp.md (revision 22235) @@ -0,0 +1,42 @@ +pstampDataStore METADATA + ds_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 + lastFileset STR 64 + outProduct STR 64 + uri STR 255 +END + +pstampProject METADATA + proj_id S64 0 # Primary Key AUTO_INCREMENT + name STR 64 # UNIQUE + state STR 64 + dbname STR 64 + dvodb STR 64 + camera STR 64 + telescope STR 64 + need_magic BOOL f +END + +pstampRequest METADATA + req_id S64 0 # Primary Key AUTO_INCREMENT + ds_id S64 0 + state STR 64 + name STR 64 # UNIQUE + reqType STR 16 + outProduct STR 64 + uri STR 255 + fault S32 0 +END + +pstampJob METADATA + job_id S64 0 # Primary Key AUTO_INCREMENT + req_id S64 0 # Primary Key fkey(req_id) ref pstampRequest(req_id) + rownum STR 64 + state STR 64 + jobType STR 16 + fault S32 0 + uri STR 255 + exp_id S64 0 + outputBase STR 255 + args STR 511 +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/skycell.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/skycell.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/skycell.md (revision 22235) @@ -0,0 +1,25 @@ +# $Id: skycell.md,v 1.4 2007-07-06 01:16:08 jhoblitt Exp $ + +skyCell METADATA + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + ra1 F64 0.0 + decl1 F64 0.0 + ra2 F64 0.0 + decl2 F64 0.0 + ra3 F64 0.0 + decl3 F64 0.0 + ra4 F64 0.0 + decl4 F64 0.0 +END + +skyCellMap METADATA + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + exp_id S64 64 # Primary Key + p3_version S32 0 # Primary Key +# class is not yet consistently carried through pXtools +# class STR 64 # Primary Key + class_id STR 64 # Primary Key +END + Index: /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_absorption.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_absorption.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_absorption.md (revision 22235) @@ -0,0 +1,25 @@ +# Table 11: Skyprobe Line Absorption Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the LRProbe observation was taken. +# Disperser ID string ID of the dispersing element +# Atm Component 1 float The strength of the 1st atmospheric component. +# Atm Component 2 float The strength of the 2nd atmospheric component. +# Atm Component 3 float The strength of the 3rd atmospheric component. +# Disperser ID string ID of the dispersing element +# Number of stars int Number of stars used to measure the absorptions. +# Astrometry coords The astrometry used on the LRProbe image. +# Exposure time float The exposure time of the LRProbe image. +# Sky brightness float The measured sky (surface) brightness, in physical units. + +skyp_absorption METADATA +# time DATETIME 2006-01-10T00:00:00 + disperser_id STR 255 + atmcomp1 F32 0.0 + atmcomp2 F32 0.0 + atmcomp3 F32 0.0 + nstars S32 0 + ra F64 0.0 + decl F64 0.0 + exptime F32 0.0 + sky_bright F64 0.0 +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_emission.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_emission.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_emission.md (revision 22235) @@ -0,0 +1,20 @@ +# Table 12: Skyprobe Line Emission Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the LRProbe observation was taken. +# Disperser ID string ID of the dispersing element +# Atm Component 1 float The strength of the 1st atmospheric component. +# Atm Component 2 float The strength of the 2nd atmospheric component. +# Atm Component 3 float The strength of the 3rd atmospheric component. +# Continuum float The strength of the continuum emission. +# Disperser ID string ID of the dispersing element +# Exposure time float The exposure time of the LRProbe image. + +skyp_emission METADATA +# time DATETIME 2006-01-11T00:00:00 + disperser_id STR 255 + atmcomp1 F32 0.0 + atmcomp2 F32 0.0 + atmcomp3 F32 0.0 + continuum F32 0.0 + exptime F32 0.0 +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_ir.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_ir.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_ir.md (revision 22235) @@ -0,0 +1,18 @@ +# Table 14: Near IR Wide-field Camera Results Table +# Column Name Datatype Description +# Time date/time The time the NIR observation was taken. +# Sky brightness float The sky (surface) brightness in the NIR observation. +# Sky variance float The variance in the sky (surface) brightness. +# Astrometry coords The astrometry used on the NIR image. +# FOV X float field width +# FOV Y float field height + +skyp_ir METADATA +# time DATETIME 2006-01-11T00:00:00 + sky_bright F64 0.0 + sky_var F64 0.0 + ra F64 0.0 + decl F64 0.0 + fov_x F32 0.0 + fov_y F32 0.0 +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_transparency.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_transparency.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/skyp_transparency.md (revision 22235) @@ -0,0 +1,21 @@ +# Table 10: SkyProbe Transparency Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the SkyProbe image was taken. +# Filter string Filter used for SkyProbe image. +# Transparency float The derived transparency. +# Number of stars int The number of stars used to measure the transparency. +# Astrometry coords The astrometry used on the SkyProbe image. +# Exposure time float The exposure time of the SkyProbe image. +# Sky brightness float The measured sky (surface) brightness, counts / second + +skyp_transparency METADATA +# time DATETIME 2006-01-10T00:00:00 # Primary Key + filter STR 255 + trans F64 0.0 + nstars S32 0 +# astrom + ra F64 0.0 + decl F64 0.0 + exptime F32 0.0 + sky_bright F64 0.0 +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/stack.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/stack.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/stack.md (revision 22235) @@ -0,0 +1,48 @@ +# $Id: stack.md,v 1.14 2008-10-07 00:01:28 price Exp $ + +stackRun METADATA + stack_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 # Key + workdir STR 255 + label STR 64 # Key + reduction STR 64 # Reduction class + dvodb STR 255 + registered TAI NULL + skycell_id STR 64 # Key + tess_id STR 64 # Key + filter STR 64 +END + +stackInputSkyfile METADATA + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) + warp_id S64 0 # Primary Key fkey(warp_id) ref warpSkyfile(warp_id) +END + +stackSumSkyfile METADATA + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) + uri STR 255 + path_base STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + dtime_stack F32 0.0 # Key + dtime_match_mean F32 0.0 # Key + dtime_match_stdev F32 0.0 # Key + dtime_initial F32 0.0 # Key + dtime_reject F32 0.0 # Key + dtime_final F32 0.0 # Key + dtime_phot F32 0.0 # Key + dtime_script F32 0.0 + match_mean F32 0.0 + match_stdev F32 0.0 + match_rms F32 0.0 + stamps_mean F32 0.0 + stamps_stdev F32 0.0 + stamps_min S32 0 + reject_images S32 0 + reject_pix_mean F32 0.0 + reject_pix_stdev F32 0.0 + sources S32 0 + hostname STR 64 + good_frac F32 0.0 # Key + fault S16 0 # Key +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/tasks.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/tasks.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/tasks.md (revision 22235) @@ -0,0 +1,241 @@ +# $Id: tasks.md,v 1.157.4.1 2008-11-09 22:08:37 price Exp $ + +# this table records all exposure ID ever seen from the summit +# exp_name == fileset + +# note that dec is a MySQL reserved word +# note that use is a MySQL reserved word +# note that exp is a MySQL reserved word + +# for use with this stored procedure to generate exp_ids under Mysql 5+ +# +#CREATE FUNCTION genTag (exp_name varchar(64)) RETURNS VARCHAR(64) +#BEGIN +# UPDATE expTagCounter SET counter = LAST_INSERT_ID(counter + 1); +# RETURN CONCAT(exp_name, '.', LAST_INSERT_ID()); +#END +# +#expTagCounter METADATA +# counter U64 0 +#END + +pzDataStore METADATA + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# list of source exposures -- updated as exposures are seen +# summitExp.imfiles is updated as filesets are queried default value should be +# -1 as NULL or 0 might be a valid value (empty fileset) +summitExp METADATA + exp_name STR 64 # Primary Key + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + dateobs UTC NULL + exp_type STR 64 + uri STR 255 + imfiles S32 0 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +# class == type of file +# class_id == type set id +# list of source images -- updated as exposures/filesets are queried +summitImfile METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope) + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + file_id STR 64 # Key + bytes S32 0 + md5sum STR 32 + class STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# list of exposures that have had their imfiles/files registered (but not +# downloaded) +pzDownloadExp METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope) + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + state STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +pzDownloadImfile METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref pzDownloadExp(exp_name, camera, telescope) + camera STR 64 # Primary Key fkey(exp_name, camera, telescope, class, class_id) ref summitImfile(exp_name, camera, telescope, class, class_id) + telescope STR 64 # Primary Key + class STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z + hostname STR 64 +END + +newExp METADATA + exp_id S64 0 # Primary Key AUTO_INCREMENT + tmp_exp_name STR 64 # Key + tmp_camera STR 64 # Key + tmp_telescope STR 64 # Key + state STR 64 # Key + workdir STR 255 # destination for output files + workdir_state STR 64 # key + reduction STR 64 # Reduction class + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + label STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +# class needs to be carried here so it can go into rawImfile and be normalized +# from there +newImfile METADATA + exp_id S64 64 # Primary Key fkey(exp_id) ref newExp(exp_id) + tmp_class_id STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# paired with rawImfile +rawExp METADATA + exp_id S64 64 # Primary Key fkey(exp_id) ref newExp(exp_id) + exp_name STR 64 # Key + camera STR 64 + telescope STR 64 + dateobs UTC 0001-01-01T00:00:00Z + exp_tag STR 255 + exp_type STR 64 + filelevel STR 64 + workdir STR 255 # destination for output files + reduction STR 64 # Reduction class + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + filter STR 64 + comment STR 80 + obs_mode STR 64 # data usage goal (eg, survey name, engineering, etc) + obs_group STR 64 # identifier for data block (eg, observation sequence) + airmass F32 0.0 + ra F64 0.0 + decl F64 0.0 + exp_time F32 0.0 + sat_pixel_frac F32 0.0 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + alt F64 0.0 + az F64 0.0 + ccd_temp F32 0.0 + posang F64 0.0 + m1_x F32 0.0 + m1_y F32 0.0 + m1_z F32 0.0 + m1_tip F32 0.0 + m1_tilt F32 0.0 + m2_x F32 0.0 + m2_y F32 0.0 + m2_z F32 0.0 + m2_tip F32 0.0 + m2_tilt F32 0.0 + env_temperature F32 0.0 + env_humidity F32 0.0 + env_wind_speed F32 0.0 + env_wind_dir F32 0.0 + teltemp_m1 F32 0.0 + teltemp_m1cell F32 0.0 + teltemp_m2 F32 0.0 + teltemp_spider F32 0.0 + teltemp_truss F32 0.0 + teltemp_extra F32 0.0 + pon_time F32 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + object STR 64 + sun_angle F32 0.0 + sun_alt F32 0.0 + moon_angle F32 0.0 + moon_alt F32 0.0 + moon_phase F32 0.0 + hostname STR 64 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +rawImfile METADATA + exp_id S64 64 # Primary Key fkey(exp_id, tmp_class_id) ref newImfile(exp_id, tmp_class_id) + exp_name STR 64 # UINDEX(exp_id, tmp_class_id) + camera STR 64 + telescope STR 64 + dateobs UTC 0001-01-01T00:00:00Z + tmp_class_id STR 64 # Key + class_id STR 64 # Primary Key + uri STR 255 + exp_type STR 64 +# This field is used to set the per exp filelevel. Thus the values for all of +# the imfiles in an exposure need to be sanity checked to make sure that this +# value is in argeement. + filelevel STR 64 + filter STR 64 + comment STR 80 + obs_mode STR 64 # data usage goal (eg, survey name, engineering, etc) + obs_group STR 64 # identifier for data block (eg, observation sequence) + airmass F32 0.0 + ra F64 0.0 + decl F64 0.0 + exp_time F32 0.0 + sat_pixel_frac F32 0.0 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + alt F64 0.0 + az F64 0.0 + ccd_temp F32 0.0 + posang F64 0.0 + m1_x F32 0.0 + m1_y F32 0.0 + m1_z F32 0.0 + m1_tip F32 0.0 + m1_tilt F32 0.0 + m2_x F32 0.0 + m2_y F32 0.0 + m2_z F32 0.0 + m2_tip F32 0.0 + m2_tilt F32 0.0 + env_temperature F32 0.0 + env_humidity F32 0.0 + env_wind_speed F32 0.0 + env_wind_dir F32 0.0 + teltemp_m1 F32 0.0 + teltemp_m1cell F32 0.0 + teltemp_m2 F32 0.0 + teltemp_spider F32 0.0 + teltemp_truss F32 0.0 + teltemp_extra F32 0.0 + pon_time F32 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + object STR 64 + sun_angle F32 0.0 + sun_alt F32 0.0 + moon_angle F32 0.0 + moon_alt F32 0.0 + moon_phase F32 0.0 + ignored BOOL FALSE + hostname STR 64 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/telescope.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/telescope.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/telescope.md (revision 22235) @@ -0,0 +1,18 @@ +# Table 16: Telescope Status +# Column Name Datatype Description +# Time date/time The time for which the telescope status is valid. +# Guide status enum The status of the guiding. +# Altitude float The telescope altitude. +# Azimuth float The telescope azimuth. +# RA float The telescope Right Ascension (ICRS ~ J2000). +# Dec float The telescope Declination (ICRS ~ J2000). + +telescope METADATA +# time DATETIME 2006-01-11T00:00:00 # Primary Key +# XXX there is currently no way to declare an enum - use str or int instead? + guide STR 255 + alt F32 0.0 + az F32 0.0 + ra F64 0.0 + decl F64 0.0 +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/warp.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/warp.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/warp.md (revision 22235) @@ -0,0 +1,58 @@ +# $Id: warp.md,v 1.19 2008-10-07 00:01:28 price Exp $ + +# +# We have at least 3 different run types +# a) single epoch differing +# b) magic run operates on "complete" exposures +# c) multiple epoch differing/stacking operations +# + +# define a new warprun for a single skycell +warpRun METADATA + warp_id S64 0 # Primary Key AUTO_INCREMENT + fake_id S64 0 # Key INDEX(warp_id, fake_id) fkey(fake_id) ref camProcessedExp(fake_id) + mode STR 64 # Key + state STR 64 # Key + workdir STR 255 + workdir_state STR 64 # Key + label STR 64 # key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + registered TAI NULL +# if magic is T then look for the exp_id in the magic output tables + magiced BOOL f # Key +END + +warpSkyCellMap METADATA + warp_id S64 0 # Primary Key fkey(warp_id) ref warpRun(warp_id) + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + class_id STR 64 # Primary Key + fault S16 0 # Key +END + +warpSkyfile METADATA + warp_id S64 0 # Primary Key fkey(warp_id, skycell_id, tess_id) ref warpSkyCellMap(warp_id, skycell_id, tess_id) + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + uri STR 255 + path_base STR 255 + data_state STR 64 # Key + bg F64 0.0 + bg_stdev F64 0.0 + dtime_warp F32 0.0 + dtime_script F32 0.0 + hostname STR 64 + good_frac F32 0.0 # Key + xmin S32 0 + xmax S32 0 + ymin S32 0 + ymax S32 0 + ignored BOOL f # Key + fault S16 0 # Key +END + +warpMask METADATA + label STR 64 # Primary Key +END Index: /branches/pap_branches/pap_branch_20081109/dbconfig/weather.md =================================================================== --- /branches/pap_branches/pap_branch_20081109/dbconfig/weather.md (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/dbconfig/weather.md (revision 22235) @@ -0,0 +1,19 @@ +# Table 9: Weather Table: some sample weather points +# Column Name Datatype Description +# Time date/time The time the weather information was measured. +# Temperature 01 float The external temperature +# Temperature 02 float The temperature at top of the dome +# Temperature 03 float The temperature on the primary mirror +# Humidity float The relative humidity. +# Pressure float The (external) atmospheric pressure. + +weather METADATA +# time DATETIME 2006-01-10T00:00:00 # Primary Key + temp01 F32 0.0 + humi01 F32 0.0 + temp02 F32 0.0 + humi02 F32 0.0 + temp03 F32 0.0 + humi03 F32 0.0 + pressure F32 0.0 +END Index: /branches/pap_branches/pap_branch_20081109/ippTools/.cvsignore =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/.cvsignore (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/.cvsignore (revision 22235) @@ -0,0 +1,16 @@ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +compile +config.guess +config.log +config.status +config.sub +configure +depcomp +install-sh +libtool +ltmain.sh +missing +pxtools.pc Index: /branches/pap_branches/pap_branch_20081109/ippTools/COPYING =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/COPYING (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/COPYING (revision 22235) @@ -0,0 +1,345 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. Index: /branches/pap_branches/pap_branch_20081109/ippTools/Makefile.am =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/Makefile.am (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/Makefile.am (revision 22235) @@ -0,0 +1,12 @@ +SUBDIRS = src share scripts + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA= pxtools.pc + +EXTRA_DIST = \ + pxtools.pc.in \ + autogen.sh + +CLEANFILES = *~ core core.* + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/pap_branches/pap_branch_20081109/ippTools/TODO =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/TODO (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/TODO (revision 22235) @@ -0,0 +1,14 @@ +- change dettol -pretend -define... to return a metadata structure that shows what the created detRun would look like +- rename dettool -residdetrun -> -todetrunsummary +- add foreign key constraints to the database +- add the ability to remove dettool detResidExps +- change p4tool to accept input from/with magic masks +- change p5tool to accept input from warp * stack +- difftool/stacktool error handling +- move all large SQL statements out into separate files +- add the option of not automatically flowing through from reg -> chip -> cam + tools +- add workdir inheritance rooted in rawExp, eg. + select chipProcessedExp.workdir, rawExp.workdir, IFNULL(chipProcessedExp.workdir, rawExp.workdir) from chipProcessedExp join rawExp using(exp_tag); +- dettool/detRun support for filelevel +- combind dettool -definebytag & -definebyquery Index: /branches/pap_branches/pap_branch_20081109/ippTools/autogen.sh =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/autogen.sh (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/autogen.sh (revision 22235) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=pxtools +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOLIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +#($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $AUTOHEADER installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" +# DIE=1 +#} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL -I m4 || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/pap_branches/pap_branch_20081109/ippTools/configure.ac =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/configure.ac (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/configure.ac (revision 22235) @@ -0,0 +1,65 @@ +AC_PREREQ(2.61) + +AC_INIT([ipptools], [1.1.36], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([autogen.sh]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDCFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +AC_PROG_LIBTOOL + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) +PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.46]) + +PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" +PXTOOLS_LIBS="${PSLIB_LIBS=} ${PSMODULES_LIBS=} ${IPPDB_LIBS=}" +AC_SUBST(PXTOOLS_CFLAGS,[$PXTOOLS_CFLAGS]) +AC_SUBST(PXTOOLS_LIBS,[$PXTOOLS_LIBS]) + +dnl check for PSDB support +TMP_CFLAGS=${CFLAGS} +CFLAGS="${CFLAGS=} ${PSLIB_CFLAGS}" + +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [[#if !defined HAVE_PSDB + error: PSDB support is not avaiable + #endif + ]])], + [], + [AC_MSG_ERROR([psLib was built without PSDB support, bailing out.])] +) + +CFLAGS=${TMP_CFLAGS} + +dnl -- check for psParseErrorCodes -------------------------------------- + +AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing]) +if test "$ERRORCODES" = "missing" ; then + AC_MSG_ERROR([psParseErrorCodes is required]) +fi + +dnl check for perl +AC_PATH_PROG([PERL], [perl], [missing]) +if test "$PERL" = "missing" ; then + AC_MSG_ERROR([perl is required]) +fi + +IPP_STDOPTS +CFLAGS="${CFLAGS=} -Wall -Werror" + +AC_CONFIG_FILES([ + Makefile + src/Makefile + share/Makefile + scripts/Makefile + pxtools.pc +]) +AC_OUTPUT Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/dettool.txt =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/dettool.txt (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/dettool.txt (revision 22235) @@ -0,0 +1,181 @@ +# returns a list of unassociated rawDetrendExp (rawDetrendExps that don't +# correspond to a exp_id in detInputExp) +dettool -pending -exp_type -inst .. + +# define a new detRun semi-automatically by selecting the component exps with +# the specified search criteria +dettool -definebyquery -det_type ... + +# create a detRun with the specified input exp_ids +dettool -definebyexp -det_type -exp_id 1 -exp_id 2 ... + +-- +# returns a list of rawImfiles that are associated with an detInputExp that do +# not appear in detProcessedImfiles +dettool -raw + +# add an imfile to detProcessedImfiles +dettool -addprocessed + +-- +# the list of class_ids that are ready to be stacked +# returns a list of det_id/iter/class_id/det_typs where class_id is unique per +# det_id (not a list of all exp_ids/class_ids) that do not have an entry in +# detStackedImfile +# results are masked by entries in detResidExpAnalysis +# may need to check detResidExp's even on iter 0 - ??? +dettool -tostack + +# list all of imfiles from different exps with the same class_id that have +# been processed +dettool -processed -wholestack -det_id ... + +# adds a stacked class_id and and optionally marks it for normalization +detool -addstacked [-pleasenormalize] -det_id -iter -class_id ... + +-- +# returns a list of det_ids/iters where the entire set of class_ids has been +# processed +dettool -tonormalize + +# returns the complete list of processed imfiles for the specified det_id +dettool -processed -unmask -det_id -iter + +# add per stacked imfile normalization statistics +# O: this could accept a MDC doc as input from stdin +dettool -addnormstat -det_id -iter -class_idi -norm F32 + +-- +# returns a list of det_ids/iter/class_id/uri from detNormStats. Stops +# returning entries after the class_is is inserted into detNormalizedImfile +dettool -normstat -det_id + +# inserts an entry into detNormalizedImfile +dettool -addnormalized -det_id -ter -class_id -uri + +-- +# returns a list of processed imfiles that have also been normalized with +# processed imfiles being masked out per detResidImfileAnalysis +# also returns stackedimfiles that have -pleasenormalize set to false +dettool -toresid -det_id -iter + +# inserts a per det_id/iter/exp_id/class_id residual +dettool -addresidimfile -det_id -exp_id -class_id -iter -stat F32 -stat_stdev F32 -uri -b1 -b2 + +# returns a list of det_id/iter/exp_ids but only for exp_ids that have resids +# for all of their imfiles +dettool -toresidexp + +# returns a list of residual imfile data +dettool -residimfile -det_id -iter -exp_id + +# inserts into detResidExpAnalysis +# sets the accept accept bool unless -reject is specified +dettool -addresidexp -det_id -iter -exp_id -jpeg1 -jpeg2 -stat F32 -stat_stdev +F32 [-reject] + +-- +# lists det_id/iter for detruns that have completed all residexps for their +# current iteration +dettool -residdetrun + +# lists all residexps for the given det_id/iter +dettool -residexp -det_id -iter + +# updates the given residexp to be accept/reject +dettool -updateresid -det_id -iter [-reject] + +# changes the state of the detrun. -rerun cause a new iteration to be started +# with the accepted with just the exp_ids +dettool -updatedetrun -det_id -iter -stat F32 -stat_stdev [-rerun|-ok|-stop] + +# manual starts a new detrun iteration starting from the specified -iter +dettool -rerun -det_id -iter -accept expid -accept expid -reject expid -reject expid +dettool -rerun -det_id -iter + +# detInputExp needs to track... +det_id iter exp_id use bool accept bool + + + +dettool + + -define | creates a a new detrend Run + input: + exp IDs? + type of det run? + creates a new detRun ID automatically + + output: the new det_id + + -raw | lists raw imfiles needing to be processed + input: + det ID + exp ID + class ID + iteration number? + output: list of unprocessed imfiles + + -addprocessed? | marks a raw imfile as processed + input: + det ID + exp ID + class ID + iteration number? + uri + stat + recipe + output: outout on error only + -processed | lists processed imfiles + input: + det ID + exp ID + class ID + iteration number? + output: list of processed imfiles + -addstacked | adds a stacked imfiles + input: + det ID + class ID + iteration number? + URI + recipe + stats + output: output on error only + -stacked | lists stacked imfiles + input: + det ID + class ID + iteration number? + output: list of stacked imfiles + -stacked | lists stacked imfiles + input: + det ID + class ID + iteration number? + output: list of stacked imfiles + -stackedframe | lists stacked imfiles for COMPLETE frames + input: + det ID // required for simplicity + iteration number? + output: list of stacked imfiles + -addmaster | adds a master imfiles + input: + det ID + class ID + iteration number? + URI + recipe + stats + output: output on error only + -master | lists master imfiles + input: + det ID // required for simplicity + class ID + iteration number? + output: a list of master imfiles + -masterframe | lists master imfiles for COMPLETE frames + input: + det ID // required for simplicity + iteration number? + output: a list of master imfiles Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/dsfilesetls =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/dsfilesetls (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/dsfilesetls (revision 22235) @@ -0,0 +1,6 @@ +#!/usr/bin/env perl + +print "# uri fileid bytes md5sum type \n"; +print "http://example.org/1 foo1.0 123 asadfasdfasdfasfdasdf CELL\n"; +print "http://example.org/2 foo1.1 123 asadfasdfasdfasdfasdf CELL\n"; +print "http://example.org/3 foo1.2 123 asadfasdfasdfasdfasdf CELL\n"; Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/dsproductls =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/dsproductls (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/dsproductls (revision 22235) @@ -0,0 +1,8 @@ +#!/usr/bin/env perl + +print "# uri fileset datetime type\n"; +print "http://example.org/1 foo1 2006-04-17T00:00:12 OBJECT\n"; +print "http://example.org/2 foo2 2006-04-17T00:00:12 OBJECT\n"; +print "http://example.org/3 foo3 2006-04-17T00:00:12 OBJECT\n"; + + Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/install.pod =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/install.pod (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/install.pod (revision 22235) @@ -0,0 +1,251 @@ +=pod + +=head1 SYNOPSIS + +C is a collection of utilties for tracking image processing +tasks. Very little decision making logic is built into these programs. The +intent is for higher level programs and scripts to make the real I +and then use C to encode those choices in a persistent manner. + +=head1 INSTALL + +=head2 Prerequisites + +C depends on C (built with C support), C, and +C and requires a C capable compiler. + +XXX document required library versions + +=head2 Build Procedure + +C is built with the so-called suite of autotools. The build +procedure is pretty typical for software built with these tools. E.g. + + ./configure + make + make install + +If you checked out the sources directly from CVS you need to run the +C script. C will run C for you so that +changes the build procedure to just: + + sh autogen.sh + make + make install + +=head2 jhbuild Build Procedure + +If you are using one of the C modulesets jhbuild can build and install C for you. E.g. + + jhbuild build pxtools + +=head2 Database Setup + +All of the C' persistent storage is ultimately done through C's +C facilties. In turn, C uses C to impliment storage (and +relational querying). C requires that each namespace (aka database) is +configured with basic access controls. These include usernames, passwords, +access filtering, and privilege restrictions. Below is a simple (but somewhat +dangerous) example of configuring a database named C. + + mysql -u root mysql -p + (enter your MySQL root password) + + DROP DATABASE IF EXISTS ipp; + CREATE DATABASE ipp; + GRANT ALL PRIVILEGES ON *.* TO 'ippuser'@'localhost' IDENTIFIED BY 'ipppass'; + FLUSH PRIVILEGES; + quit + +After setting up the database you should verify that you can connect to with the username and password you configured + + mysql -u ipp ipp -pipp + +Next, C needs to be told where to to find MySQL, the name of the +database you created, the username, and the password you created. C +uses the typical C configuration system. The simplest method is to +add the follow lines to your $HOME/.ipprc file. + + ### Database configuration + DBSERVER STR localhost # Database host name (for psDBInit) + DBNAME STR ipp + DBUSER STR ipp # Database user name (for psDBInit) + DBPASSWORD STR ipp # Database password (for psDBInit) + +The C utilities require the a schema is pre-loaded into the database. The C tool can do this for you. E.g. + + pxadmin -create + +You should always verify that the scheme was actually loaded. E.g. + + mysql -u ipp ipp -pipp + Welcome to the MySQL monitor. Commands end with ; or \g. + Your MySQL connection id is 6 to server version: 4.1.20-log + + Type 'help;' or '\h' for help. Type '\c' to clear the buffer. + + mysql> show tables; + +------------------------+ + | Tables_in_ipp | + +------------------------+ + | detInputExp | + | detMasterFrame | + | detMasterImfile | + | detNormalizedImfile | + | detProcessedImfile | + | detResidExpAnalysis | + | detResidImfileAnalysis | + | detRun | + | detStackedImfile | + | newExp | + | newImfile | + | p1PendingExp | + | p2DoneExp | + | p2DoneImfile | + | p2PendingExp | + | p2PendingImfile | + | p3PendingExp | + | pzPendingExp | + | pzPendingImfile | + | rawDetrendExp | + | rawImfile | + | rawScienceExp | + | summitExp | + +------------------------+ + 23 rows in set (0.00 sec) + +=head1 USE + +The workflow is as follows: + +XXX + +=head2 Phase 0 tools + +The primary C utility for handling Phase 0 is C. This tool +has two modes C<-pending> and C<-update>. The pending mode lists all +Cs that are part of an exposure that I yet been classified +as either science or detrend data. + +For example: + + p0search -pending + +Would output something like: + + newImfile MULTI # + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 0 + uri STR file://0 + END + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 1 + uri STR file://1 + END + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 2 + uri STR file://2 + END + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 3 + uri STR file://3 + END + +To declare C as science data, you would invoke C like this: + + p0search -update -exp_id 10 + +Note that by default C<-update> flags the exposure as science data. To declare it to be detrend data the command would be: + + p0search -update -exp_id 10 -detrend + +Also keep in mind that this is a one time declaration. Once it has been +declared as either science or detrend data this decision can not be undone +(without manually editing the database) + +=head2 Phase 1 tools + +These tools are not yet in use. + +=head2 Phase 2 tools + +C handles all of state for Phase 2. It has four basic modes: + +=over 4 + +=item * C<-quick> + +=item * C<-define> + +=item * C<-pending> + + p0search -pending + +=item * C<-done> + +=head2 Administrative tools + +=over 4 + +=item * C + +This tool can be used to load/unload the database schema. It has three modes of operation: + +=over 4 + +=item * C<-create> + +Loads the database schema. + +=item * C<-delete> + +Deletes the database schema. + +=item * C<-recreate> + +Deletes the database schema then reloads it. + +=back + +=item * C + +I data directly into the database. It has one mode parameter for each database table. Note: support for only a few tables has been impliemented. + +=back + +=head2 Bypassing 'Summit Copy' + +The C suite assumes that exposures and image data are initially +registered with the C set of I tools. Sometimes this is +inconvenient as you want to work with local data and bypass this set. This is +what the C utility is for. C directly inserts data into +the database, side stepping any ordering rules or sanity checking. + +The end result of the C utilities is that new exposures are registered in the C table and raw image data is in the C table. You can insert this data yourself, preparing the system to run I. + +Here is an example of inserting an exposure that has 4 component image files. + + #!//bin/sh + + inject="./pxinject" + + $inject -newExp -exp_id 10 -inst gpc -telescope ps1 -exp_type object -imfiles 4 + + `$inject -newImfile -exp_id 10 -class OTA -class_id 0 -uri file://0` + `$inject -newImfile -exp_id 10 -class OTA -class_id 1 -uri file://1` + `$inject -newImfile -exp_id 10 -class OTA -class_id 2 -uri file://2` + `$inject -newImfile -exp_id 10 -class OTA -class_id 3 -uri file://3` + +=cut Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/magic_flow.txt =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/magic_flow.txt (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/magic_flow.txt (revision 22235) @@ -0,0 +1,65 @@ +### Set up a run: +magictool -definerun -workdir /path/to/workdir +magictool -addinputskyfile -magic_id 6 -diff_id 1 -node 5.1 +magictool -addinputskyfile -magic_id 6 -diff_id 2 -node 5.2 +magictool -addinputskyfile -magic_id 6 -diff_id 3 -node 5.3 +magictool -addinputskyfile -magic_id 6 -diff_id 4 -node 5.4 +magictool -updaterun -magic_id 6 -state run + +### Set up a run in a single call: +magictool -queue -select_XXX + +### Generate a processing tree +magictool -totree +#magic_tree.pl --magic_id X [camera, tess_id, ra0, dec0] +* magictool -inputskyfile -magic_id 6 +* magictool -inputtree -magic_id 6 -dep_file magic_dep.mdc + +### Process the tree +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR 5.1 +# dep STR NULL +# done S64 0 +# uri STR NULL +#END +#magic_process.pl --magic_id 6 --node 5.1 +magictool -inputs -magic_id 6 -node 5.1 + +magictool -addresult -magic_id 6 -node 5.1 -uri foo +magictool -addresult -magic_id 6 -node 5.2 -uri bar +magictool -addresult -magic_id 6 -node 5.3 -uri baz +magictool -addresult -magic_id 6 -node 5.4 -uri qix + +### Merging the tree +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR 5A +# uri MULTI +# uri STR foo +# uri STR bar +#END +#magic_process.pl --magic_id 6 --node 5A +magictool -inputs -magic_id 6 -node 5A + +magictool -addresult -magic_id 6 -node 5A -uri foobar +magictool -addresult -magic_id 6 -node 5B -uri bazqix + +### Final merge +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR root +#END +#magic_process.pl --magic_id 6 --node root +magictool -inputs -magic_id 6 -node root + +magictool -addresult -magic_id 6 -node root -uri foobarbazqix + +### Generate the mask description +magictool -tomask +magic_mask.pl --magic_id X +magictool -inputs -magic_id 6 -node root +magictool -addmask -magic_id 6 -uri foobarbazqixqit Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/p0tools.txt =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/p0tools.txt (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/p0tools.txt (revision 22235) @@ -0,0 +1,40 @@ + +Phase 0 pipeline tools: + +p0search -pending : + * examine the new.imfiles,new.exposures tables and select exposures ready for analysis + * output is: (expID) (camera) + +p0search -update (expID) (camera): + * select a the corresponding images from the new.imfiles/new.exposures table + * extract the specified header information + * search the summit metadata db tables + * write an entry to the raw.imfiles and raw.exposure tables + * set the state on the new.imfiles,new.exposure tables + * based on the camera config information; + * add an entry to the p1.pending table (mosaic) + - or + * add an entry to the p2.pending table (single) + +p0search -stats (expID) (camera): + * select a specified image in the new.imfiles/new.exposures table + * extract the specified header information + * search the summit metadata db tables + * report the image stats + [-update without output to MDDB] + +p0search -mkraw (expID) (camera): + * select a specified image in the new.imfiles/new.exposures table + * extract the specified header information + * search the summit metadata db tables + * write an entry to the raw.imfiles and raw.exposure tables + * set the state on the new.imfiles,new.exposure tables + +p0search -cleanup: + * remove completed entries from the new.imfiles,new.exposure tables + +** note : the division of -pending and -update allows separate processes + to be examining the image headers and measuring some stats. these + jobs can be run via pcontrol to reduce the load on the PanTasks + machines + Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/p1tools.txt =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/p1tools.txt (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/p1tools.txt (revision 22235) @@ -0,0 +1,16 @@ + +Phase 1 pipeline tools: + +p1search -define [constraints]: + * examine the raw.exposures tables and select exposures matching the given criteria + * add entries which are allowed (mosaic) to the p1.pending table + +p1search -pending : + * examine the p1.pending table and select exposures waiting for p1 + * output: lines consisting of: + (expID) (p1version) (camera) + +p1search -done : + * select completed entries in the p1.pending table + * move to the p1.done table + * add new entry to the p2.pending tables Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/p2search.sh =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/p2search.sh (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/p2search.sh (revision 22235) @@ -0,0 +1,8 @@ +#!/bin/csh -f + +echo "# expID class classID url" +echo "654320o fpa megacam 654320o.fits" +echo "654321o fpa megacam 654321o.fits" +echo "654322o fpa megacam 654322o.fits" +echo "654323o fpa megacam 654323o.fits" +echo "654324o fpa megacam 654324o.fits" Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/p2tools.txt =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/p2tools.txt (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/p2tools.txt (revision 22235) @@ -0,0 +1,41 @@ + +Phase 2 pipeline tools: + +p2search -quick + * search for images which match in raw.exp,raw.imfiles + * output in format which can be used by ppImage pantasks script + +p2search -define [options] + * input: searches mddb:raw_exposures,raw_images + - compares against p2Pending[Exp|Imfile] + - compares against p2Done[Exp|Imfile] + - XXX should be implemented as a special wrapper function that + implements this comparison as a SQL query + * output: updates mddb:P2_exposures_pending,P2_images_pending + * alternative output: identical to p2pending + +p2search -pending + * input: searches mddb:P2_exposures_pending,P2_images_pending + * output: Nlines consisting of: + (URL) (expID) (class) + * options: ? + +p2search -update + * examine the imfiles and identify any completed exposures + +p2search -done + * get exp_id/class/class_id from the CLI + * add the completed imfile to the p2DoneImfile tables + * remove corresponding entries from the p2PendingImfile table + * check to see if any p2PendingExps have no associated p2PendingImfiles + if so move the p2PendingExp(s) to p2DoneExp +// * send new entry to the pending p3 table + +ppImage file://path/filename file://path/outroot -recipe (recipe) +ppImage neb://nebname neb://outroot -recipe (recipe) + +restriction options: + -time (start) (stop) + -camera (camera) + -region (ra,dec) (ra,dec) + Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/p3tools.txt =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/p3tools.txt (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/p3tools.txt (revision 22235) @@ -0,0 +1,22 @@ + +Phase 3 pipeline tools: + +p3search -define : + * examine the raw.exposures tables and select exposures matching the given criteria + * add entries which are allowed (mosaic) to the p3.pending table + +p3search -quick : + * examine the raw.exposures tables and select exposures matching the given criteria + * return list of entries for p3 processing + * output: lines consisting of: + (expID) (p3version) (camera) + +p3search -pending : + * examine the p3.pending exposures table and select exposures waiting for p3 + * return list of entries for p3 processing + * output: lines consisting of: + (expID) (p3version) (camera) + +p3search -done : + * select completed entries in the p3.pending table + * move to the p3.done table Index: /branches/pap_branches/pap_branch_20081109/ippTools/doc/summit_copy.txt =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/doc/summit_copy.txt (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/doc/summit_copy.txt (revision 22235) @@ -0,0 +1,49 @@ +The summit copy phase (phase Z or pz) consists of three basic steps. +Discovering what exposures and available, discovering what image files are +associated with those exposures, and the downloading of image files. All +"external" communication with the DataStore is done with three utilities; +pzgetexp, pzgetimfiles & dsget. While local state information is accessed and +updated via the 'pztool' utility. + +The basic work follow is: + +pzgetexp retrieves a list of all of the exposures (filesets) available from a +DataStore "product". If any previously unknown file exposures are found they +are added to the pzPendingExp tables and the summitExp table. New entries to +the summitExp table have a imfile value of NULL as this information can't yet +be known. If any exposures are already in the summitExp table the most recent +exp_id is used to constrain the result set requested from the DataStore. + + ./pzgetexp -uri http://otis/ds/skyprobe/ -inst skyprobe -telescope ps1 + +The results of pzgetexp and accessible with pztool -pendingexp`. Which will +return a list of new exposures from the pzPendingExp table. + + ./pztool -pendingexp + +The output of which is used to invoke pzgetimfiles. + +pzgetimfiles retrieves a list of all of the imfiles (files) in an exposure +(fileset) from the DataStore. It then updates the imfiles count in the +summitExp table and adds each imfile to the pzPendingImfile table. + +./pzgetimfiles -uri http://otis/ds/skyprobe/sep8_twi_05/ -filesetid sep8_twi_05 -inst skyprobe -telescope ps1 + +pztool -pendingimfile returns the list of imfiles that need to be downloaded. + + ./pztool -pendingimfile + +The results of which are used to invoke dsget and pztool -copydone after dsget +completes it's download. + +dsget from the 'DataStore' package takes the follow parameters. + + dsget --uri --filename [--bytes ] [--md5 ] + +Where "filename" may be either just a filename relative to the CWD or a fully +qualified path. After dsget has copied the file pztool -copydone needs to be +invoked + + ./pztool -copydone -exp_id sep8_twi_03 -inst skyprobe -telescope ps1 -class chip -class_id sep8_twi_03.fits -uri file///data/alala/sep8_twi_03.fits + +And then we're Pau. Index: /branches/pap_branches/pap_branch_20081109/ippTools/m4/ac_prog_perl_modules.m4 =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/m4/ac_prog_perl_modules.m4 (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/m4/ac_prog_perl_modules.m4 (revision 22235) @@ -0,0 +1,53 @@ +dnl @synopsis AC_PROG_PERL_MODULES([MODULES], [ACTION-IF-TRUE], [ACTION-IF-FALSE]) +dnl +dnl Checks to see if the the given perl modules are available. If true +dnl the shell commands in ACTION-IF-TRUE are executed. If not the shell +dnl commands in ACTION-IF-FALSE are run. Note if $PERL is not set (for +dnl example by calling AC_CHECK_PROG, or AC_PATH_PROG), +dnl AC_CHECK_PROG(PERL, perl, perl) will be run. +dnl +dnl Example: +dnl +dnl AC_CHECK_PERL_MODULES(Text::Wrap Net::LDAP, , +dnl AC_MSG_WARN(Need some Perl modules) +dnl +dnl @category InstalledPackages +dnl @author Dean Povey +dnl @version 2002-09-25 +dnl @license AllPermissive + +AC_DEFUN([AC_PROG_PERL_MODULES],[ + ac_perl_modules="$1" + # Make sure we have perl + if test -z "$PERL"; then + AC_CHECK_PROG(PERL,perl,perl) + fi + + if test "x$PERL" != x; then + ac_perl_modules_failed=0 + for ac_perl_module in $ac_perl_modules; do + AC_MSG_CHECKING(for perl module $ac_perl_module) + + # Would be nice to log result here, but can't rely on autoconf + # internals + $PERL "-M$ac_perl_module" -e exit > /dev/null 2>&1 + if test $? -ne 0; then + AC_MSG_RESULT(no); + ac_perl_modules_failed=1 + else + AC_MSG_RESULT(ok); + fi + done + + # Run optional shell commands + if test "$ac_perl_modules_failed" = 0; then + : + $2 + else + : + $3 + fi + else + AC_MSG_WARN(could not find perl) + fi +]) Index: /branches/pap_branches/pap_branch_20081109/ippTools/notes.txt =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/notes.txt (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/notes.txt (revision 22235) @@ -0,0 +1,63 @@ + +2008.07.19 EAM + + * need to add cleanup modes to all stages. here is the minimum list + of stages that need the cleanups: + + * chiptool - DONE + * camtool + * faketool + * warptool + * difftool + * stacktool + * dettool processedimfile + * dettool processedexp + * dettool stackedimfile + * dettool normalizedstat + * dettool normalizedimfile + * dettool normalizedexp + * dettool residimfile + * dettool residexp + + * I am modifiying the tools to use the new states: new, full, goto_cleaned, etc. + I am NOT changing: regtool, pztool, magictool, flatcorr, pxregister, pstamptool + + * for dettool, I have added a new field, 'data_state' to the + detRunSummary. My plan here is that a complete detRunIteration + is cleaned at once. I've added functions to all of the dettool + major modes to list pending entries to be cleaned and a function + to update the data_state when cleaning is done. + +2008.07.11 EAM + + * break dettool.c into files for each detrend stage + * require -iteration in more places? (esp -add...) + * allow %s, %d arguments in the *.sql files or not? + * adddetrunsummary has the ability to set values for multiple detRunSummary entries. why? + +2008.07.09 EAM + + In order to implement the 'cleanup' and 'update' strategies, we have + changed the state names and defined addtional states as follows: + + * new : entry is unprocessed (was 'run') + * full : entry is processed and has all output data products (was 'done') + * goto_cleaned : entry should be processed by the 'cleanup' system + * cleaned : entry has been processed by the 'cleanup' system -- + some output data products are now missing. + + * goto_full : entry should be re-processed by the 'update' system + * full : entry has been re-processed by the 'update' system + + * goto_purged : entry should be processed by the 'cleanup' ssytem + to purge all output data + * purged : entry has been purged + + + +2008.05.16 EAM + +caltool + * add the active field, -active flag to -dbs + * add the -region + * change 'catdir' to 'dvo_id' Index: /branches/pap_branches/pap_branch_20081109/ippTools/pxtools.pc.in =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/pxtools.pc.in (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/pxtools.pc.in (revision 22235) @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@/@PACKAGE_NAME@ + +Name: @PACKAGE_NAME@ +Description: Pan-STARRS IPP Tools Library +Version: @VERSION@ +Requires: pslib psmodules ippdb +Libs: -L${libdir} -lpxtools +Cflags: -I${includedir} @PXTOOLS_CFLAGS@ Index: /branches/pap_branches/pap_branch_20081109/ippTools/scripts/.cvsignore =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/scripts/.cvsignore (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/scripts/.cvsignore (revision 22235) @@ -0,0 +1,2 @@ +Makefile +Makefile.in Index: /branches/pap_branches/pap_branch_20081109/ippTools/scripts/Makefile.am =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/scripts/Makefile.am (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/scripts/Makefile.am (revision 22235) @@ -0,0 +1,3 @@ + +bin_SCRIPTS = \ + ippadmin Index: /branches/pap_branches/pap_branch_20081109/ippTools/scripts/camtest.sh =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/scripts/camtest.sh (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/scripts/camtest.sh (revision 22235) @@ -0,0 +1,11 @@ +#!//bin/sh + +set -o verbose + +chiptest.sh || exit 1 + +camtool -pendingexp || exit 1 +camtool -pendingimfile || exit 1 + +camtool -addprocessedexp -cam_id 1 -uri file:///cam -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -sigma_ra 1 -sigma_dec 2 -zp_mean 10 -zp_stdev 2 -n_stars 2 -n_extended 0 -n_astrom 42 -n_cr 10000000 -path_base file:///foo || exit 1 +camtool -addprocessedexp -cam_id 2 -uri file:///cam -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -sigma_ra 1 -sigma_dec 2 -zp_mean 10 -zp_stdev 2 -n_stars 2 -n_extended 0 -n_astrom 42 -n_cr 10000000 -path_base file:///foo || exit 1 Index: /branches/pap_branches/pap_branch_20081109/ippTools/scripts/chiptest.sh =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/scripts/chiptest.sh (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/scripts/chiptest.sh (revision 22235) @@ -0,0 +1,17 @@ +#!//bin/sh + +set -o verbose + +regtest.sh || exit 1 + +chiptool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + chiptool -addprocessedimfile -chip_id 1 -exp_id 1 -class_id $ID -uri file://chipp-t10.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1 +done; + +for ID in `seq 0 3`; do + chiptool -addprocessedimfile -chip_id 2 -exp_id 2 -class_id $ID -uri file://chipp-t11.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1 +done; + +chiptool -pendingimfile || exit 1 Index: /branches/pap_branches/pap_branch_20081109/ippTools/scripts/dettest.sh =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/scripts/dettest.sh (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/scripts/dettest.sh (revision 22235) @@ -0,0 +1,88 @@ +#!//bin/sh + +set -o verbose + +det_id=1 + +./regtest.sh -end_stage reg || exit 1 + +det_id=`dettool -definebyquery -det_type bias -inst gpc -filelevel fpa -select_exp_type bias -airmass_min 1 -airmass_max 10 -exp_time_min 10 -exp_time_max 30.0 -workdir file::///some/path -simple | cut -f1 -d" "` || exit 1 + +dettool -raw || exit 1 + +dettool -toprocessedimfile || exit 1 + +for ID in `seq 0 3` ; do + dettool -addprocessedimfile -det_id 1 -exp_id 1 -class_id $ID -uri file://proc-$ID -recip myrecip -bg 2 -bg_stdev 3 -bg_mean_stdev 4 || exit 1 +done; + +for ID in `seq 0 3` ; do + dettool -addprocessedimfile -det_id 1 -exp_id 2 -class_id $ID -uri file://proc-$ID -recip myrecip -bg 2 -bg_stdev 3 -bg_mean_stdev 4 || exit 1 +done; + + +dettool -tostacked || exit 1 + +for ID in `seq 0 3` ; do + dettool -addstacked -det_id $det_id -uri file://stacked-$ID -class_id $ID -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 || exit 1 +done; + +dettool -tonormalizedstat || exit 1 + +for ID in `seq 0 3` ; do + dettool -addnormalizedstat -det_id $det_id -class_id $ID -norm 0.12345 || exit 1 +done; + +dettool -tonormalize || exit 1 +for ID in `seq 0 3` ; do + dettool -addnormalizedimfile -det_id $det_id -class_id $ID -uri file://normalized-$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base banana1 || exit 1 +done; + +dettool -tonormalizedexp || exit 1 +dettool -addnormalizedexp -det_id $det_id -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file://normalizedexp || exit 1 + +dettool -toresidimfile || exit 1 + +for ID in `seq 0 3` ; do + dettool -addresidimfile -det_id $det_id -exp_id 1 -class_id $ID -recip myrecip -bg 1 -bg_stdev 1 -bg_mean_stdev 1 -uri file://resid-$ID || exit 1 +done; + +for ID in `seq 0 3` ; do + dettool -addresidimfile -det_id $det_id -exp_id 2 -class_id $ID -recip myrecip -bg 1 -bg_stdev 1 -bg_mean_stdev 1 -uri file://resid-$ID || exit 1 +done; + +dettool -toresidexp || exit 1 +dettool -addresidexp -det_id $det_id -exp_id 1 -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base jpeg1 || exit 1 +dettool -addresidexp -det_id $det_id -exp_id 2 -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base jpeg1 -reject || exit 1 + +dettool -todetrunsummary || exit 1 +dettool -residexp || exit 1 +dettool -updateresidexp -det_id $det_id -iteration 0 -recip yourrecipe || exit 1 +dettool -updateresidexp -det_id $det_id -iteration 0 -exp_id 2 -reject || exit 1 +dettool -updateresidexp -det_id $det_id -iteration 0 -exp_id 2 || exit 1 + +dettool -adddetrunsummary -det_id $det_id -iteration 0 -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -accept || exit 1 + +dettool -todetrunsummary || exit 1 + +dettool -residexp || exit 1 + +dettool -updatedetrun -det_id $det_id -state stop || exit 1 + +dettool -register_detrend -det_type bias -mode master -filelevel fpa -workdir file:/// -label foo || exit 1 +dettool -register_detrend_imfile -det_id 2 -class_id 1 -uri file:///foo || exit 1 +dettool -register_detrend_imfile -det_id 2 -class_id 2 -uri file:///foo || exit 1 +dettool -register_detrend_imfile -det_id 2 -class_id 3 -uri file:///foo || exit 1 +dettool -register_detrend_imfile -det_id 2 -class_id 4 -uri file:///foo || exit 1 +dettool -updatedetrun -det_id 2 -state stop || exit 1 + +# correct test +dettool -makecorrection -det_id 1 || exit 1 +dettool -tocorrectexp || exit 1 +dettool -tocorrectimfile -det_id 3 || exit 1 +dettool -addcorrectimfile -det_id 3 -class_id 0 -uri file:///correct/0 || exit 1 +dettool -addcorrectimfile -det_id 3 -class_id 1 -uri file:///correct/1 || exit 1 +dettool -addcorrectimfile -det_id 3 -class_id 2 -uri file:///correct/2 || exit 1 +dettool -addcorrectimfile -det_id 3 -class_id 3 -uri file:///correct/3 || exit 1 +# detRun 3 should be automatically set to stop by this point +dettool -tocorrectexp || exit 1 Index: /branches/pap_branches/pap_branch_20081109/ippTools/scripts/difftest.sh =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/scripts/difftest.sh (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/scripts/difftest.sh (revision 22235) @@ -0,0 +1,29 @@ +#!//bin/sh + +set -o verbose + +warptest.sh || exit 1 + + +difftool -definerun -workdir file:///tmp/diff -skycell_id foo1 -tess_id bar || exit 1 +difftool -addinputskyfile -diff_id 1 -warp_id 1 -kind warped -template || exit 1 +difftool -addinputskyfile -diff_id 1 -warp_id 1 -kind warped || exit 1 +#difftool -updaterun -state run -diff_id 1 || exit 1 +difftool -todiffskyfile || exit 1 +difftool -inputskyfile || exit 1 +difftool -adddiffskyfile -diff_id 1 -uri file:///tmp/diff/skyfile -path_base file://lalaland -bg 1 -bg_stdev 2 || exit 1 +difftool -diffskyfile -diff_id 1 || exit 1 +difftool -updaterun -state stop -diff_id 1 || exit 1 + +# +# diff_id 2 +# +difftool -definerun -workdir file:///tmp/diff -skycell_id foo1 -tess_id bar || exit 1 +difftool -addinputskyfile -diff_id 2 -warp_id 1 -kind warped -template || exit 1 +difftool -addinputskyfile -diff_id 2 -warp_id 1 -kind warped || exit 1 +#difftool -updaterun -state run -diff_id 1 || exit 1 +difftool -todiffskyfile || exit 1 +difftool -inputskyfile || exit 1 +difftool -adddiffskyfile -diff_id 2 -uri file:///tmp/diff/skyfile -path_base file://lalaland -bg 1 -bg_stdev 2 || exit 1 +difftool -diffskyfile -diff_id 2 || exit 1 +difftool -updaterun -state stop -diff_id 2 || exit 1 Index: /branches/pap_branches/pap_branch_20081109/ippTools/scripts/faketest.sh =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/scripts/faketest.sh (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/scripts/faketest.sh (revision 22235) @@ -0,0 +1,19 @@ +#!//bin/sh + +set -o verbose + +camtest.sh || exit 1 + +#faketool -pendingexp || exit 1 +faketool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + faketool -addprocessedimfile -fake_id 1 -exp_id 1 -class_id $ID -uri file://chipp-t10.$ID -path_base file:///foo || exit 1 +done; + +for ID in `seq 0 3`; do + faketool -addprocessedimfile -fake_id 2 -exp_id 2 -class_id $ID -uri file://chipp-t11.$ID -path_base file:///foo || exit 1 +done; + +faketool -pendingimfile || exit 1 +faketool -processedimfile || exit 1 Index: /branches/pap_branches/pap_branch_20081109/ippTools/scripts/ippadmin =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/scripts/ippadmin (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/scripts/ippadmin (revision 22235) @@ -0,0 +1,77 @@ +#!/bin/csh -f +## this script creates the tables needed by ippMonitor for user administration + +if ($#argv < 4) goto usage; + +if ("$1" == "init") then + if ($#argv != 4) goto usage; + set dbserver = $2 + set dbuser = $3 + set dbpass = $4 + + mysql -h $dbserver -u root -p <= 0.5 +-- magictool_definebyquery_temp_insert_groupby.sql +GROUP BY + exp_id, + warpSkyfile.skycell_id +; + + +-- magictool_definebyquery_select_part1.sql +-- This is part 1 of 2 of a query to get a list of exposures on which magic may be performed +-- After this follows magictool_definebyquery_select_part2.sql +SELECT + exp_id, + filter, + num_todo, + num_done +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id, tess_id) + JOIN camRun USING(chip_id, tess_id) + JOIN fakeRun USING(cam_id, tess_id) + JOIN warpRun USING(fake_id, tess_id) + JOIN warpSkyCellMap USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'full' + -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +----- AND warpSkyfile.good_frac > 0.5 +-- INSERT HERE magictool_definebyquery_select_part2.sql +-- magictool_definebyquery_select_part2.sql +-- This is part 2 of 2 of a query to get a list of exposures on which magic may be performed +-- This follows magictool_definebyquery_select_part1.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL +; + +-- magictool_definebyquery_insert.sql +-- Insert the best list of diffs as magic inputs +INSERT INTO magicInputSkyfile +SELECT + @MAGIC_ID@, -- Update this with the appropriate magic_id + diff_id, + CONCAT(tess_id, '.', skycell_id) AS node +FROM magicBestDiffs +; Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_insert.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_insert.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_insert.sql (revision 22235) @@ -0,0 +1,9 @@ +-- Insert the best list of diffs as magic inputs +INSERT INTO magicInputSkyfile +SELECT + @MAGIC_ID@, -- Update this with the appropriate magic_id + diff_id, + skycell_id +FROM magicBestDiffs +WHERE + exp_id = @EXP_ID@ -- Update this with the appropriate exp_id Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_select_part1.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_select_part1.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_select_part1.sql (revision 22235) @@ -0,0 +1,25 @@ +-- This is part 1 of 2 of a query to get a list of exposures on which magic may be performed +-- After this follows magictool_definebyquery_select_part2.sql +SELECT + exp_id, + filter, + num_todo, + num_done +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(DISTINCT skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id) + JOIN camRun USING(chip_id) + JOIN fakeRun USING(cam_id) + JOIN warpRun USING(fake_id) + JOIN warpSkyCellMap USING(warp_id) + JOIN warpSkyfile USING(warp_id, skycell_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'full' +-- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +-- INSERT HERE magictool_definebyquery_select_part2.sql Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_select_part2.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_select_part2.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_select_part2.sql (revision 22235) @@ -0,0 +1,18 @@ +-- This is part 2 of 2 of a query to get a list of exposures on which magic may be performed +-- This follows magictool_definebyquery_select_part1.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_select_test.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_select_test.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_select_test.sql (revision 22235) @@ -0,0 +1,46 @@ +-- This is the combination of the two parts of the query to get a list +-- of exposures to be magic-ed + +-------------------------------------------------------------------------------- +-- THIS FILE IS INTENDED FOR TESTING ONLY! IT IS NOT USED BY stacktool. +-- CHANGES SHOULD BE MADE TO magictool_definebyquery_select_part1.sql +-- AND magictool_definebyquery_select_part2.sql +-------------------------------------------------------------------------------- + +-- magictool_definebyquery_select_part1.sql +SELECT + * +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id) + JOIN camRun USING(chip_id) + JOIN fakeRun USING(cam_id) + JOIN warpRun USING(fake_id) + JOIN warpSkyCellMap USING(warp_id) + JOIN warpSkyfile USING(warp_id, skycell_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'full' + -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +-- magictool_definebyquery_select_part2.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_temp_create.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_temp_create.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_temp_create.sql (revision 22235) @@ -0,0 +1,8 @@ +CREATE TEMPORARY TABLE magicBestDiffs ( +exp_id BIGINT, +skycell_id VARCHAR(64), +tess_id VARCHAR(64), +warp_id BIGINT, +diff_id BIGINT, +PRIMARY KEY(exp_id, skycell_id, tess_id) +) ENGINE=MEMORY; Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_temp_insert.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_temp_insert.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_temp_insert.sql (revision 22235) @@ -0,0 +1,22 @@ +-- List of best differences for each exposure +INSERT INTO magicBestDiffs +SELECT + rawExp.exp_id, + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + warpRun.warp_id AS warp_id, + MAX(diffSkyfile.diff_id) AS diff_id +FROM rawExp +JOIN chipRun USING(exp_id) +JOIN camRun USING(chip_id) +JOIN fakeRun USING(cam_id) +JOIN warpRun USING(fake_id) +JOIN warpSkyCellMap USING(warp_id) +JOIN warpSkyfile USING(warp_id, skycell_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.template = 0 -- selecting inputs only +JOIN diffSkyfile USING(diff_id) +WHERE + diffSkyfile.fault = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql (revision 22235) @@ -0,0 +1,3 @@ +GROUP BY + exp_id, + warpSkyfile.skycell_id Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_diffskyfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_diffskyfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_diffskyfile.sql (revision 22235) @@ -0,0 +1,13 @@ +SELECT + diff_id, + diffSkyfile.uri, + diffSkyfile.path_base +FROM magicRun +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile USING(diff_id) +WHERE + diffRun.state = 'full' + AND diffSkyfile.fault = 0 +-- AND magicRun.state = 'stop' +-- AND magicMask.fault = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_inputs.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_inputs.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_inputs.sql (revision 22235) @@ -0,0 +1,33 @@ +SELECT * +FROM ( +-- Single skycells +SELECT + magicRun.magic_id, + magicRun.state, + magicInputSkyfile.node, + diffSkyfile.uri, + diffSkyfile.path_base, + diffSkyfile.fault +FROM magicInputSkyfile +JOIN diffSkyfile + USING(diff_id) +JOIN magicRun + USING(magic_id) +UNION +-- Merged skycells +SELECT + magicRun.magic_id, + magicRun.state, + magicTree.node, + magicNodeResult.uri, + NULL, -- magicNodeResult doesn't have a path_base + magicNodeResult.fault +FROM magicTree +JOIN magicRun + USING(magic_id) +JOIN magicNodeResult + ON magicTree.magic_id = magicNodeResult.magic_id + AND magicTree.dep = magicNodeResult.node +) as Foo +WHERE + fault = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_inputskyfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_inputskyfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_inputskyfile.sql (revision 22235) @@ -0,0 +1,7 @@ +SELECT + magicInputSkyfile.* +FROM magicInputSkyfile +JOIN magicRun + USING(magic_id) +WHERE + magicRun.state = 'run' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_mask.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_mask.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_mask.sql (revision 22235) @@ -0,0 +1,8 @@ +SELECT + magicMask.* +FROM magicMask +JOIN magicRun + USING(magic_id) +WHERE + magicRun.state = 'stop' + AND magicMask.fault = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_rawimfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_rawimfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_rawimfile.sql (revision 22235) @@ -0,0 +1,24 @@ +SELECT + exp_id, + class_id, + rawImfile.uri +FROM magicRun +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffRun.diff_id + -- Want input warps only + AND diffInputSkyfile.warp_id IS NOT NULL + AND diffInputSkyfile.template = 0 +JOIN warpSkyfile USING(warp_id,skycell_id) +JOIN warpRun USING(warp_id) +JOIN warpSkyCellMap USING(warp_id,skycell_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN rawImfile USING(exp_id,class_id) +WHERE + rawImfile.fault = 0 +-- AND magicRun.state = 'stop' +-- AND magicMask.fault = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_tomask.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_tomask.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_tomask.sql (revision 22235) @@ -0,0 +1,18 @@ +SELECT + magic_id, + exp_id, + camera, + magicRun.workdir, + uri +FROM magicRun +JOIN rawExp USING(exp_id) +JOIN magicTree USING(magic_id) +LEFT JOIN magicNodeResult USING(magic_id, node) +WHERE + magicRun.state = 'run' + AND magicNodeResult.node = 'root' + AND magicNodeResult.fault = 0 +GROUP BY + magic_id +HAVING + COUNT(magicTree.node) = COUNT(magicNodeResult.node) Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_toprocess_inputs.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_toprocess_inputs.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_toprocess_inputs.sql (revision 22235) @@ -0,0 +1,19 @@ +SELECT + magicTree.*, + rawExp.camera, + diffSkyfile.path_base, + -- convert magic_id into a boolean value (1 or 0) + -- note that the type stays a 64 bit int + magicNodeResult.magic_id IS TRUE as done +FROM magicTree +JOIN magicRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id, node) +JOIN diffSkyfile USING(diff_id) +JOIN rawExp USING(exp_id) +LEFT JOIN magicNodeResult + ON magicTree.magic_id = magicNodeResult.magic_id + AND magicTree.node = magicNodeResult.node +WHERE + magicRun.state = 'run' + AND magicNodeResult.magic_id IS NULL + AND magicNodeResult.node IS NULL Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_toprocess_tree.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_toprocess_tree.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_toprocess_tree.sql (revision 22235) @@ -0,0 +1,17 @@ +SELECT + magicTree.*, + exp_id, + camera, + magicRun.workdir, + -- convert magic_id into a boolean value (1 or 0) + -- note that the type stays a 64 bit int + magicNodeResult.magic_id IS TRUE as done, + magicNodeResult.fault IS TRUE as bad +FROM magicTree +JOIN magicRun USING(magic_id) +JOIN rawExp USING(exp_id) +LEFT JOIN magicNodeResult + ON magicTree.magic_id = magicNodeResult.magic_id + AND magicTree.node = magicNodeResult.node +WHERE + magicRun.state = 'run' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_toskyfilemask.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_toskyfilemask.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_toskyfilemask.sql (revision 22235) @@ -0,0 +1,10 @@ +SELECT + magicMask.* +FROM magicRun +JOIN magicMask + USING(magic_id) +LEFT JOIN magicSkyfileMask + USING(magic_id) +WHERE + magicRun.state = 'run' + AND magicSkyfileMask.magic_id IS NULL Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_totree.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_totree.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_totree.sql (revision 22235) @@ -0,0 +1,16 @@ +SELECT + magic_id, + exp_id, + camera, + magicRun.workdir, + ra, + decl, + tess_id +FROM magicRun +JOIN rawExp USING(exp_id) +LEFT JOIN magicTree + USING(magic_id) +WHERE + magicRun.state = 'run' + AND magicTree.node IS NULL + AND magicRun.fault = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_warpskyfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_warpskyfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/magictool_warpskyfile.sql (revision 22235) @@ -0,0 +1,20 @@ +SELECT + warp_id, + skycell_id, + warpSkyfile.uri, + warpSkyfile.path_base +FROM magicRun +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffRun.diff_id + -- Want input warps only + AND diffInputSkyfile.warp_id IS NOT NULL + AND diffInputSkyfile.template = 0 +JOIN warpSkyfile USING(warp_id,skycell_id) +JOIN warpRun USING(warp_id) +WHERE + warpRun.state = 'full' + AND warpSkyfile.fault = 0 +-- AND magicRun.state = 'stop' +-- AND magicMask.fault = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_addjob_otherjob.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_addjob_otherjob.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_addjob_otherjob.sql (revision 22235) @@ -0,0 +1,3 @@ +INSERT INTO pstampJob + (req_id, rownum, state, jobType, uri, exp_id, outputBase, fault) + VALUES( %lld, '%s', '%s', '%s', '%s', %lld, '%s', %s) Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_addjob_stampjob.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_addjob_stampjob.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_addjob_stampjob.sql (revision 22235) @@ -0,0 +1,3 @@ +INSERT INTO pstampJob + (req_id, rownum, state, jobType, uri, exp_id, outputBase, fault, args) + VALUES( %lld, '%s', '%s', '%s', '%s', %lld, '%s', %s, '%s') Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_datastore.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_datastore.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_datastore.sql (revision 22235) @@ -0,0 +1,1 @@ +SELECT * FROM pstampDataStore Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_pendingjob.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_pendingjob.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_pendingjob.sql (revision 22235) @@ -0,0 +1,4 @@ +SELECT + * + FROM pstampJob + WHERE state = 'run' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_pendingreq.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_pendingreq.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_pendingreq.sql (revision 22235) @@ -0,0 +1,4 @@ +SELECT + * + FROM pstampRequest + WHERE state = 'new' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_project.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_project.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pstamptool_project.sql (revision 22235) @@ -0,0 +1,1 @@ +SELECT * FROM pstampProject Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pxadmin_create_tables.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pxadmin_create_tables.sql (revision 22235) @@ -0,0 +1,1170 @@ +CREATE TABLE pzDataStore ( + camera VARCHAR(64), + telescope VARCHAR(64), + uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(camera, telescope) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE summitExp ( + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + exp_type VARCHAR(64), + uri VARCHAR(255), + imfiles INT, + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_name, camera, telescope), + KEY(fault)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE summitImfile ( + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + file_id VARCHAR(64), + bytes INT, + md5sum VARCHAR(32), + class VARCHAR(64), + class_id VARCHAR(64), + uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_name, camera, telescope, class, class_id), + KEY(file_id), + FOREIGN KEY (exp_name, camera, telescope) + REFERENCES summitExp(exp_name, camera, telescope)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pzDownloadExp ( + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + state VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_name, camera, telescope), + KEY(state), + FOREIGN KEY (exp_name, camera, telescope) + REFERENCES summitExp(exp_name, camera, telescope)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pzDownloadImfile ( + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + class VARCHAR(64), + class_id VARCHAR(64), + uri VARCHAR(255), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + hostname VARCHAR(64), + PRIMARY KEY(exp_name, camera, telescope, class, class_id), + KEY(fault), + FOREIGN KEY (exp_name, camera, telescope) + REFERENCES pzDownloadExp(exp_name, camera, telescope), + FOREIGN KEY (exp_name, camera, telescope, class, class_id) + REFERENCES summitImfile(exp_name, camera, telescope, class, class_id)) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE newExp ( + exp_id BIGINT AUTO_INCREMENT, + tmp_exp_name VARCHAR(64), + tmp_camera VARCHAR(64), + tmp_telescope VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + label VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id), + KEY(exp_id), + KEY(tmp_exp_name), + KEY(tmp_camera), + KEY(tmp_telescope), + KEY(state), + KEY(workdir_state), + KEY(end_stage), + KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE newImfile ( + exp_id BIGINT, + tmp_class_id VARCHAR(64), + uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id, tmp_class_id), + FOREIGN KEY (exp_id) + REFERENCES newExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rawExp ( + exp_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + exp_tag VARCHAR(255), + exp_type VARCHAR(64), + filelevel VARCHAR(64), + workdir VARCHAR(255), + reduction VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + filter VARCHAR(64), + comment VARCHAR(80), + obs_mode VARCHAR(64), + obs_group VARCHAR(64), + airmass FLOAT, + ra DOUBLE, + decl DOUBLE, + exp_time FLOAT, + sat_pixel_frac FLOAT, + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + alt DOUBLE, + az DOUBLE, + ccd_temp FLOAT, + posang DOUBLE, + m1_x FLOAT, + m1_y FLOAT, + m1_z FLOAT, + m1_tip FLOAT, + m1_tilt FLOAT, + m2_x FLOAT, + m2_y FLOAT, + m2_z FLOAT, + m2_tip FLOAT, + m2_tilt FLOAT, + env_temperature FLOAT, + env_humidity FLOAT, + env_wind_speed FLOAT, + env_wind_dir FLOAT, + teltemp_m1 FLOAT, + teltemp_m1cell FLOAT, + teltemp_m2 FLOAT, + teltemp_spider FLOAT, + teltemp_truss FLOAT, + teltemp_extra FLOAT, + pon_time FLOAT, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + object VARCHAR(64), + sun_angle FLOAT, + sun_alt FLOAT, + moon_angle FLOAT, + moon_alt FLOAT, + moon_phase FLOAT, + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id), + KEY(exp_name), + KEY(end_stage), + KEY(fault), + FOREIGN KEY (exp_id) + REFERENCES newExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rawImfile ( + exp_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + tmp_class_id VARCHAR(64), + class_id VARCHAR(64), + uri VARCHAR(255), + exp_type VARCHAR(64), + filelevel VARCHAR(64), + filter VARCHAR(64), + comment VARCHAR(80), + obs_mode VARCHAR(64), + obs_group VARCHAR(64), + airmass FLOAT, + ra DOUBLE, + decl DOUBLE, + exp_time FLOAT, + sat_pixel_frac FLOAT, + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + alt DOUBLE, + az DOUBLE, + ccd_temp FLOAT, + posang DOUBLE, + m1_x FLOAT, + m1_y FLOAT, + m1_z FLOAT, + m1_tip FLOAT, + m1_tilt FLOAT, + m2_x FLOAT, + m2_y FLOAT, + m2_z FLOAT, + m2_tip FLOAT, + m2_tilt FLOAT, + env_temperature FLOAT, + env_humidity FLOAT, + env_wind_speed FLOAT, + env_wind_dir FLOAT, + teltemp_m1 FLOAT, + teltemp_m1cell FLOAT, + teltemp_m2 FLOAT, + teltemp_spider FLOAT, + teltemp_truss FLOAT, + teltemp_extra FLOAT, + pon_time FLOAT, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + object VARCHAR(64), + sun_angle FLOAT, + sun_alt FLOAT, + moon_angle FLOAT, + moon_alt FLOAT, + moon_phase FLOAT, + ignored TINYINT DEFAULT 0, + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id, class_id), + KEY(tmp_class_id), + KEY(fault), + UNIQUE KEY(exp_id, tmp_class_id), + FOREIGN KEY (exp_id, tmp_class_id) + REFERENCES newImfile(exp_id, tmp_class_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE guidePendingExp (guide_id BIGINT AUTO_INCREMENT, exp_id BIGINT, recipe VARCHAR(64), PRIMARY KEY(guide_id), KEY(guide_id), KEY(exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipRun ( + chip_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + PRIMARY KEY(chip_id), + KEY(chip_id), KEY(exp_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(chip_id, exp_id), + FOREIGN KEY (exp_id) + REFERENCES rawExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipProcessedImfile ( + chip_id BIGINT, + exp_id BIGINT, + class_id VARCHAR(64), + data_state VARCHAR(64), + uri VARCHAR(255), + bg FLOAT, + bg_stdev FLOAT, + bg_mean_stdev FLOAT, + bias FLOAT, + bias_stdev FLOAT, + fringe_0 FLOAT, + fringe_1 FLOAT, + fringe_2 FLOAT, + ap_resid FLOAT, + ap_resid_stdev FLOAT, + fwhm_major FLOAT, + fwhm_major_lq FLOAT, + fwhm_major_uq FLOAT, + fwhm_minor FLOAT, + fwhm_minor_lq FLOAT, + fwhm_minor_uq FLOAT, + iq_fwhm_major FLOAT, + iq_fwhm_major_err FLOAT, + iq_fwhm_minor FLOAT, + iq_fwhm_minor_err FLOAT, + iq_m2 FLOAT, + iq_m2_err FLOAT, + iq_m2_lq FLOAT, + iq_m2_uq FLOAT, + iq_m2c FLOAT, + iq_m2c_err FLOAT, + iq_m2c_lq FLOAT, + iq_m2c_uq FLOAT, + iq_m2s FLOAT, + iq_m2s_err FLOAT, + iq_m2s_lq FLOAT, + iq_m2s_uq FLOAT, + iq_m3 FLOAT, + iq_m3_err FLOAT, + iq_m3_lq FLOAT, + iq_m3_uq FLOAT, + iq_m4 FLOAT, + iq_m4_err FLOAT, + iq_m4_lq FLOAT, + iq_m4_uq FLOAT, + dtime_detrend FLOAT, + dtime_photom FLOAT, + dtime_total FLOAT, + dtime_script FLOAT, + hostname VARCHAR(64), + n_stars INT, + n_psfstars INT, + n_iqstars INT, + n_extended INT, + n_cr INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(chip_id, exp_id, class_id), + KEY(data_state), + KEY(fault), + FOREIGN KEY (chip_id, exp_id) + REFERENCES chipRun(chip_id, exp_id), + FOREIGN KEY (exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camRun ( + cam_id BIGINT AUTO_INCREMENT, + chip_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + PRIMARY KEY(cam_id), + KEY(cam_id), + KEY(chip_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(cam_id, chip_id), + FOREIGN KEY (chip_id) + REFERENCES chipRun(chip_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camProcessedExp ( + cam_id BIGINT, + uri VARCHAR(255), + bg FLOAT, + bg_stdev FLOAT, + bg_mean_stdev FLOAT, + bias FLOAT, + bias_stdev FLOAT, + fringe_0 FLOAT, + fringe_1 FLOAT, + fringe_2 FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + ap_resid FLOAT, + ap_resid_stdev FLOAT, + zpt_obs FLOAT, + zpt_stdev FLOAT, + zpt_lq FLOAT, + zpt_uq FLOAT, + fwhm_major FLOAT, + fwhm_major_lq FLOAT, + fwhm_major_uq FLOAT, + fwhm_minor FLOAT, + fwhm_minor_lq FLOAT, + fwhm_minor_uq FLOAT, + iq_fwhm_major FLOAT, + iq_fwhm_major_err FLOAT, + iq_fwhm_minor FLOAT, + iq_fwhm_minor_err FLOAT, + iq_m2 FLOAT, + iq_m2_err FLOAT, + iq_m2_lq FLOAT, + iq_m2_uq FLOAT, + iq_m2c FLOAT, + iq_m2c_err FLOAT, + iq_m2c_lq FLOAT, + iq_m2c_uq FLOAT, + iq_m2s FLOAT, + iq_m2s_err FLOAT, + iq_m2s_lq FLOAT, + iq_m2s_uq FLOAT, + iq_m3 FLOAT, + iq_m3_err FLOAT, + iq_m3_lq FLOAT, + iq_m3_uq FLOAT, + iq_m4 FLOAT, + iq_m4_err FLOAT, + iq_m4_lq FLOAT, + iq_m4_uq FLOAT, + dtime_script FLOAT, + dtime_astrom FLOAT, + dtime_addstar FLOAT, + hostname VARCHAR(64), + n_stars INT, + n_psfstars INT, + n_iqstars INT, + n_extended INT, + n_cr INT, + n_astrom INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(cam_id), + KEY(fault), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeRun ( + fake_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id), + KEY(cam_id), + KEY(state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(fake_id, cam_id), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeProcessedImfile ( + fake_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + dtime_fake FLOAT, + dtime_script FLOAT, + hostname VARCHAR(64), + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id), + FOREIGN KEY (exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detRun ( + det_id BIGINT AUTO_INCREMENT, + iteration INT, + det_type VARCHAR(64), + mode VARCHAR(64), + state VARCHAR(64), + filelevel VARCHAR(64), + workdir VARCHAR(255), + camera VARCHAR(64), + telescope VARCHAR(64), + exp_type VARCHAR(64), + reduction VARCHAR(64), + filter VARCHAR(64), + airmass_min FLOAT, + airmass_max FLOAT, + exp_time_min FLOAT, + exp_time_max FLOAT, + ccd_temp_min FLOAT, + ccd_temp_max FLOAT, + posang_min DOUBLE, + posang_max DOUBLE, + registered DATETIME, + time_begin DATETIME, + time_end DATETIME, + use_begin DATETIME, + use_end DATETIME, + solang_min FLOAT, + solang_max FLOAT, + label VARCHAR(64), + ref_det_id BIGINT, + ref_iter INT, + -- parent INT, :: dropping this + PRIMARY KEY(det_id), + KEY(det_id), + KEY(iteration), + KEY(det_type), + KEY(mode), + KEY(state), + KEY(label), + -- KEY(parent), :: dropping this + INDEX(det_id, iteration)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detInputExp ( +det_id BIGINT, +iteration INT, +exp_id BIGINT, +include TINYINT, +PRIMARY KEY(det_id, iteration, exp_id), +INDEX(det_id, exp_id), +INDEX(det_id, iteration), +FOREIGN KEY (det_id) +REFERENCES detRun(det_id), +FOREIGN KEY (exp_id) +REFERENCES rawExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detProcessedImfile ( + det_id BIGINT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id, class_id), + KEY(fault), + INDEX(det_id, class_id), + INDEX(det_id, exp_id), + FOREIGN KEY (det_id, exp_id) + REFERENCES detInputExp(det_id, exp_id), + FOREIGN KEY (exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detProcessedExp ( + det_id BIGINT, + exp_id BIGINT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id), + KEY(fault), + FOREIGN KEY (det_id, exp_id) + REFERENCES detInputExp(det_id, exp_id), + FOREIGN KEY (det_id, exp_id) + REFERENCES detProcessedImfile(det_id, exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detStackedImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, class_id) + REFERENCES detProcessedImfile(det_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedStatImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + norm FLOAT, + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, iteration, class_id) + REFERENCES detStackedImfile(det_id, iteration, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + INDEX(det_id, iteration), + FOREIGN KEY (det_id) + REFERENCES detInputExp(det_id), + FOREIGN KEY (det_id, iteration, class_id) + REFERENCES detNormalizedStatImfile(det_id, iteration, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedExp ( + det_id BIGINT, + iteration INT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, iteration) + REFERENCES detNormalizedImfile(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detResidImfile ( + det_id BIGINT, + iteration INT, + ref_det_id BIGINT, + ref_iter INT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + bg_skewness DOUBLE, + bg_kurtosis DOUBLE, + bin_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + fringe_resid_0 DOUBLE, + fringe_resid_1 DOUBLE, + fringe_resid_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, exp_id, class_id), + KEY(fault), + INDEX(det_id, iteration, exp_id), + FOREIGN KEY (det_id, iteration, exp_id) + REFERENCES detInputExp(det_id, iteration, exp_id), + FOREIGN KEY (det_id, exp_id, class_id) + REFERENCES detProcessedImfile(det_id, exp_id, class_id), + FOREIGN KEY (ref_det_id, ref_iter) + REFERENCES detNormalizedExp(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detResidExp ( + det_id BIGINT, + iteration INT, + exp_id BIGINT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + bg_skewness DOUBLE, + bg_kurtosis DOUBLE, + bin_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + fringe_resid_0 DOUBLE, + fringe_resid_1 DOUBLE, + fringe_resid_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + accept TINYINT, + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, exp_id), + KEY(fault), + INDEX(det_id, iteration), + FOREIGN KEY (det_id, iteration, exp_id) + REFERENCES detInputExp(det_id, iteration, exp_id), + FOREIGN KEY (det_id, iteration, exp_id) + REFERENCES detResidImfile(det_id, iteration, exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detRunSummary ( + det_id BIGINT, + iteration INT, + data_state VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + accept TINYINT, + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, iteration) + REFERENCES detResidExp(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detRegisteredImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detRun(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpRun ( + warp_id BIGINT AUTO_INCREMENT, + fake_id BIGINT, + mode VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + registered DATETIME, + magiced TINYINT, + PRIMARY KEY(warp_id), + KEY(warp_id), + KEY(fake_id), + KEY(mode), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(end_stage), + KEY(magiced), + INDEX(warp_id, fake_id), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyCellMap ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + class_id VARCHAR(64), + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id, class_id), + KEY(fault), + FOREIGN KEY (warp_id) + REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyfile ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + uri VARCHAR(255), + path_base VARCHAR(255), + data_state VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_warp FLOAT, + dtime_script FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + xmin INT, + xmax INT, + ymin INT, + ymax INT, + ignored TINYINT, + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id), + KEY(good_frac), + KEY(ignored), KEY(fault), + FOREIGN KEY (warp_id, skycell_id, tess_id) + REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackRun ( + stack_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + filter VARCHAR(64), + PRIMARY KEY(stack_id), + KEY(stack_id), + KEY(state), + KEY(skycell_id), + KEY(tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackInputSkyfile ( + stack_id BIGINT, + warp_id BIGINT, + PRIMARY KEY(stack_id, warp_id), + FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id), + FOREIGN KEY (warp_id) REFERENCES warpSkyfile(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackSumSkyfile ( + stack_id BIGINT, + uri VARCHAR(255), + path_base VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_stack FLOAT, + dtime_match_mean FLOAT, + dtime_match_stdev FLOAT, + dtime_initial FLOAT, + dtime_reject FLOAT, + dtime_final FLOAT, + dtime_phot FLOAT, + dtime_script FLOAT, + match_mean FLOAT, + match_stdev FLOAT, + match_rms FLOAT, + stamps_mean FLOAT, + stamps_stdev FLOAT, + stamps_min INT, + reject_images INT, + reject_pix_mean FLOAT, + reject_pix_stdev FLOAT, + sources INT, + hostname VARCHAR(64), + good_frac FLOAT, + fault SMALLINT, + PRIMARY KEY(stack_id), + KEY(dtime_stack), + KEY(good_frac), + KEY(fault), + FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffRun ( + diff_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + PRIMARY KEY(diff_id), + KEY(diff_id), + KEY(state), + KEY(skycell_id), + KEY(tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffInputSkyfile ( + diff_id BIGINT, + template TINYINT, + stack_id BIGINT, + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + kind VARCHAR(64), + PRIMARY KEY(diff_id, template), + KEY(stack_id), + KEY(warp_id), + KEY(skycell_id), + KEY(tess_id), + KEY(kind), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id), + FOREIGN KEY (stack_id) REFERENCES stackSumSkyfile(stack_id), + FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffSkyfile ( + diff_id BIGINT, + uri VARCHAR(255), + path_base VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + stamps_num INT, + stamps_mean FLOAT, + stamps_rms FLOAT, + norm FLOAT, + bg_diff FLOAT, + kernel_x FLOAT, + kernel_y FLOAT, + kernel_xx FLOAT, + kernel_xy FLOAT, + kernel_yy FLOAT, + sources INT, + dtime_diff FLOAT, + dtime_match FLOAT, + dtime_phot FLOAT, + dtime_script FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + fault SMALLINT, + PRIMARY KEY(diff_id), + KEY(good_frac), + KEY(fault), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicRun ( + magic_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(255), + label VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + fault SMALLINT, + PRIMARY KEY(magic_id), + KEY(magic_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(fault), + FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicInputSkyfile ( + magic_id BIGINT, + diff_id BIGINT, + node VARCHAR(64), + PRIMARY KEY(magic_id, diff_id), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicTree ( + magic_id BIGINT, + node VARCHAR(64), + dep VARCHAR(64), + KEY(magic_id), + KEY(node), + KEY(dep), + INDEX(magic_id, node), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicNodeResult ( + magic_id BIGINT, + node VARCHAR(64), + uri VARCHAR(255), + fault SMALLINT, + PRIMARY KEY(magic_id, node), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY (magic_id, node) REFERENCES magicTree(magic_id, node), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicMask ( + magic_id BIGINT, + uri VARCHAR(255), + streaks INT, + fault SMALLINT, + PRIMARY KEY(magic_id), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE calDB ( + cal_id BIGINT AUTO_INCREMENT, + dvodb VARCHAR(64), + state VARCHAR(64), + PRIMARY KEY(cal_id), + KEY(cal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE calRun ( + cal_id BIGINT AUTO_INCREMENT, + region VARCHAR(64), + last_step VARCHAR(64), + state VARCHAR(64), + PRIMARY KEY(cal_id), + KEY(cal_id), + KEY(last_step), + FOREIGN KEY (cal_id) REFERENCES calDB(cal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrRun ( + corr_id BIGINT AUTO_INCREMENT, + det_type VARCHAR(64), + dvodb VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + filter VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + region VARCHAR(64), + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(corr_id), + KEY(corr_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- these two tables link the flatcorrRun to the associated chip and camera runs +CREATE TABLE flatcorrChipLink ( + corr_id BIGINT, + chip_id BIGINT, + include TINYINT, + PRIMARY KEY(corr_id, chip_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrCamLink ( + corr_id BIGINT, + chip_id BIGINT, + cam_id BIGINT, + include TINYINT, + PRIMARY KEY(corr_id, cam_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id), + FOREIGN KEY (cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampDataStore ( + ds_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + lastFileset VARCHAR(64), + outProduct VARCHAR(64) UNIQUE, + uri VARCHAR(255), + PRIMARY KEY(ds_id), + KEY(ds_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampProject ( + proj_id BIGINT AUTO_INCREMENT, + name VARCHAR(64) UNIQUE, + state VARCHAR(64), + dbname VARCHAR(64), + dvodb VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + need_magic TINYINT, + PRIMARY KEY(proj_id), + KEY(proj_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampRequest ( + req_id BIGINT AUTO_INCREMENT, + ds_id BIGINT, + state VARCHAR(64), + name VARCHAR(64) UNIQUE, + reqType VARCHAR(16), + outProduct VARCHAR(64), + uri VARCHAR(255), + fault SMALLINT, + PRIMARY KEY(req_id), + KEY(req_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampJob ( + job_id BIGINT AUTO_INCREMENT, + req_id BIGINT, + rownum VARCHAR(64), + state VARCHAR(64), + jobType VARCHAR(16), + fault SMALLINT, + uri VARCHAR(255), + exp_id BIGINT, + outputBase VARCHAR(255), + args VARCHAR(511), + PRIMARY KEY(job_id, req_id), + KEY(job_id), + FOREIGN KEY (req_id) REFERENCES pstampRequest(req_id) +) ENGINE=innodb DEFAULT CHARSET=latin1 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pxadmin_drop_tables.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pxadmin_drop_tables.sql (revision 22235) @@ -0,0 +1,58 @@ +SET FOREIGN_KEY_CHECKS=0; +DROP TABLE IF EXISTS pzDataStore; +DROP TABLE IF EXISTS summitExp; +DROP TABLE IF EXISTS summitImfile; +DROP TABLE IF EXISTS pzDownloadExp; +DROP TABLE IF EXISTS pzDownloadImfile; +DROP TABLE IF EXISTS newExp; +DROP TABLE IF EXISTS newImfile; +DROP TABLE IF EXISTS rawExp; +DROP TABLE IF EXISTS rawImfile; +DROP TABLE IF EXISTS guidePendingExp; +DROP TABLE IF EXISTS chipRun; +DROP TABLE IF EXISTS chipProcessedImfile; +DROP TABLE IF EXISTS chipMask; +DROP TABLE IF EXISTS camRun; +DROP TABLE IF EXISTS camProcessedExp; +DROP TABLE IF EXISTS camMask; +DROP TABLE IF EXISTS detRun; +DROP TABLE IF EXISTS detInputExp; +DROP TABLE IF EXISTS fakeRun; +DROP TABLE IF EXISTS fakeProcessedImfile; +DROP TABLE IF EXISTS fakeMask; +DROP TABLE IF EXISTS detProcessedImfile; +DROP TABLE IF EXISTS detProcessedExp; +DROP TABLE IF EXISTS detStackedImfile; +DROP TABLE IF EXISTS detNormalizedStatImfile; +DROP TABLE IF EXISTS detNormalizedImfile; +DROP TABLE IF EXISTS detNormalizedExp; +DROP TABLE IF EXISTS detResidImfile; +DROP TABLE IF EXISTS detResidExp; +DROP TABLE IF EXISTS detRunSummary; +DROP TABLE IF EXISTS detRegisteredImfile; +DROP TABLE IF EXISTS warpRun; +DROP TABLE IF EXISTS warpSkyCellMap; +DROP TABLE IF EXISTS warpSkyfile; +DROP TABLE IF EXISTS warpMask; +DROP TABLE IF EXISTS diffRun; +DROP TABLE IF EXISTS diffInputSkyfile; +DROP TABLE IF EXISTS diffSkyfile; +DROP TABLE IF EXISTS stackRun; +DROP TABLE IF EXISTS stackInputSkyfile; +DROP TABLE IF EXISTS stackSumSkyfile; +DROP TABLE IF EXISTS magicRun; +DROP TABLE IF EXISTS magicInputSkyfile; +DROP TABLE IF EXISTS magicTree; +DROP TABLE IF EXISTS magicNodeResult; +DROP TABLE IF EXISTS magicMask; +DROP TABLE IF EXISTS magicSkyfileMask; +DROP TABLE IF EXISTS calDB; +DROP TABLE IF EXISTS calRun; +DROP TABLE IF EXISTS flatcorrRun; +DROP TABLE IF EXISTS flatcorrChipLink; +DROP TABLE IF EXISTS flatcorrCamLink; +DROP TABLE IF EXISTS pstampDataStore; +DROP TABLE IF EXISTS pstampProject; +DROP TABLE IF EXISTS pstampRequest; +DROP TABLE IF EXISTS pstampJob; +SET FOREIGN_KEY_CHECKS=1 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_find_completed_exp.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_find_completed_exp.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_find_completed_exp.sql (revision 22235) @@ -0,0 +1,35 @@ +SELECT DISTINCT + exp_name, -- return should match pzDownloadExp + camera, + telescope, + state, + NULL as epoch -- epoch +FROM ( + SELECT + pzDownloadImfile.*, + pzDownloadExp.state, + summitExp.imfiles + FROM pzDownloadExp + JOIN summitExp + USING(exp_name, camera, telescope) + LEFT JOIN pzDownloadImfile + USING(exp_name, camera, telescope) + LEFT JOIN newExp + ON pzDownloadExp.exp_name = newExp.tmp_exp_name + AND pzDownloadExp.camera = newExp.tmp_camera + AND pzDownloadExp.telescope = newExp.tmp_telescope + WHERE + pzDownloadExp.state = 'run' + AND newExp.tmp_exp_name IS NULL + AND newExp.tmp_camera IS NULL + AND newExp.tmp_telescope IS NULL + GROUP BY + pzDownloadExp.exp_name, + pzDownloadExp.camera, + pzDownloadExp.telescope + -- it doesn't matter which field in pzDownloadImfile we count as we've + -- already Download a group by + HAVING + COUNT(pzDownloadImfile.exp_name) = summitExp.imfiles + AND SUM(pzDownloadImfile.fault) = 0 + ) as Foo Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_pendingimfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_pendingimfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_pendingimfile.sql (revision 22235) @@ -0,0 +1,20 @@ +SELECT * +FROM ( + SELECT + summitImfile.*, + summitExp.dateobs + FROM summitImfile + JOIN pzDownloadExp + USING(exp_name, camera, telescope) + LEFT JOIN pzDownloadImfile + USING(exp_name, camera, telescope, class, class_id) + JOIN summitExp + USING(exp_name, camera, telescope) + WHERE + pzDownloadExp.state = 'run' + AND pzDownloadImfile.exp_name IS NULL + AND pzDownloadImfile.camera IS NULL + AND pzDownloadImfile.telescope IS NULL + AND pzDownloadImfile.class IS NULL + AND pzDownloadImfile.class_id IS NULL +) as Foo Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_revert_downloadimfile_faults.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_revert_downloadimfile_faults.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_revert_downloadimfile_faults.sql (revision 22235) @@ -0,0 +1,17 @@ +DELETE FROM pzDownloadImfile +WHERE + fault != 0 + -- fault was logged in the last 3 days + AND ABS(TIMESTAMPDIFF(SECOND, epoch, NOW())) < 86400 * 3 + AND ( + -- HTTP 503: try again + fault = 203 + -- HTTP 500: timeout + OR fault = 200 + -- HTTP 404: unknown datastore internal problem + OR fault = 104 + -- ipptool errors + OR fault < 100 + -- perl untrapped die() + OR fault = 255 + ) Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_revert_fileset_faults.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_revert_fileset_faults.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_revert_fileset_faults.sql (revision 22235) @@ -0,0 +1,17 @@ +UPDATE summitExp SET fault = 0 +WHERE + fault != 0 + -- fault is older than 10 minutes + AND ABS(TIMESTAMPDIFF(SECOND, epoch, NOW())) > 60 * 10 + -- fault was logged in the last 3 days + AND ABS(TIMESTAMPDIFF(SECOND, epoch, NOW())) < 86400 * 3 + AND ( + -- HTTP 500: timeout + fault = 200 + -- HTTP 404: unknown datastore internal problem + OR fault = 104 + -- ipptool errors + OR fault < 100 + -- perl untrapped die() + OR fault = 255 + ) Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_revertcopied.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_revertcopied.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/pztool_revertcopied.sql (revision 22235) @@ -0,0 +1,3 @@ +DELETE FROM pzDownloadImfile +WHERE + fault != 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_create_dup_table.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_create_dup_table.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_create_dup_table.sql (revision 22235) @@ -0,0 +1,2 @@ +CREATE TEMPORARY TABLE duplicate (exp_id BIGINT, PRIMARY KEY(exp_id)) +ENGINE=MEMORY Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_pendingexp.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_pendingexp.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_pendingexp.sql (revision 22235) @@ -0,0 +1,49 @@ +SELECT DISTINCT + exp_id, + tmp_exp_name, + tmp_camera, + tmp_telescope, + state, + workdir, + workdir_state, + reduction, + dvodb, + tess_id, + end_stage, + label, + camera, + filter, + obs_mode, + obs_group, + epoch, + dateobs +FROM + (SELECT + newExp.*, + newImfile.tmp_class_id, + rawImfile.tmp_class_id as raw_tmp_class_id, + rawImfile.camera, + rawImfile.filter, + rawImfile.dateobs, + rawImfile.obs_mode, + rawImfile.obs_group + FROM newExp + JOIN newImfile + USING(exp_id) + LEFT JOIN rawExp + USING(exp_id) + LEFT JOIN rawImfile + ON newImfile.exp_id = rawImfile.exp_id + AND newImfile.tmp_class_id = rawImfile.tmp_class_id + WHERE + newExp.state = 'run' + AND rawExp.exp_id IS NULL + AND rawImfile.ignored = 0 +-- where hook %s + GROUP BY + newExp.exp_id + HAVING + COUNT(newImfile.tmp_class_id) = COUNT(rawImfile.tmp_class_id) + AND SUM(rawImfile.fault) = 0 +-- limit hook %s + ) as Foo Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_pendingimfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_pendingimfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_pendingimfile.sql (revision 22235) @@ -0,0 +1,17 @@ +SELECT DISTINCT + newExp.exp_id, + newExp.tmp_exp_name, + newExp.tmp_camera, + newExp.tmp_telescope, + newExp.workdir, + newImfile.tmp_class_id, + newImfile.uri +FROM newImfile +JOIN newExp + USING(exp_id) +LEFT JOIN rawImfile + ON newExp.exp_id = rawImfile.exp_id + AND newImfile.tmp_class_id = rawImfile.tmp_class_id +WHERE + newExp.state = 'run' + AND rawImfile.tmp_class_id IS NULL Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_populate_dup_table.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_populate_dup_table.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_populate_dup_table.sql (revision 22235) @@ -0,0 +1,13 @@ +INSERT INTO duplicate +SELECT exp_id FROM + ( + SELECT + MAX(exp_id) AS exp_id, +-- tmp_exp_name, +-- tmp_camera, +-- tmp_telescope, + count(exp_id) AS count + FROM newExp + GROUP BY tmp_exp_name, tmp_camera, tmp_telescope + HAVING count > 1 + ) as Foo Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_processedimfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_processedimfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_processedimfile.sql (revision 22235) @@ -0,0 +1,9 @@ +SELECT + rawImfile.*, + newExp.tmp_exp_name, + newExp.tmp_camera, + newExp.tmp_telescope +FROM rawImfile +JOIN newExp + USING(exp_id) +WHERE rawImfile.ignored = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_revertprocessedexp.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_revertprocessedexp.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_revertprocessedexp.sql (revision 22235) @@ -0,0 +1,3 @@ +DELETE FROM rawExp +WHERE + fault != 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_revertprocessedimfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_revertprocessedimfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/regtool_revertprocessedimfile.sql (revision 22235) @@ -0,0 +1,3 @@ +DELETE FROM rawImfile +WHERE + fault != 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_insert.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_insert.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_insert.sql (revision 22235) @@ -0,0 +1,23 @@ +-- Define the inputs to a defined stack +INSERT INTO + stackInputSkyfile(stack_id, warp_id) +SELECT + @STACK_ID@, -- This should be replaced with the stack_id + warpRun.warp_id +FROM warpSkyfile +JOIN warpRun + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +WHERE + warpSkyfile.skycell_id = '%s' + AND warpRun.state = 'full' + AND rawExp.filter = '%s' + AND warpSkyfile.fault = 0 + AND warpSkyfile.ignored = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_insert_random_part1.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_insert_random_part1.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_insert_random_part1.sql (revision 22235) @@ -0,0 +1,32 @@ +-- This is the part 1 of 2 of a query to use a random set of warps for +-- the inputs to a defined stack. +-- stacktool_definebyquery_insert_random_part2.sql should be appended. +INSERT INTO + stackInputSkyfile(stack_id, warp_id) +SELECT + @STACK_ID@, -- This should be replaced with the stack_id + warp_id +FROM ( + -- Sub-select to get the random set of warps + SELECT + warpSkyfile.*, + rand() AS rnd_num + FROM warpSkyfile + JOIN warpRun + USING(warp_id) + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + skycell_id = '%s' + AND warpRun.state = 'full' + AND rawExp.filter = '%s' -- the result of the query is grouped by filter and inserted for one at a time + AND warpSkyfile.fault = 0 + AND warpSkyfile.ignored = 0 +-- Put additional constraints here +-- stacktool_definebyquery_insert_random_part2.sql goes here Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_insert_random_part2.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_insert_random_part2.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_insert_random_part2.sql (revision 22235) @@ -0,0 +1,8 @@ +-- This is the part 2 of 2 of a query to use a random set of warps for +-- the inputs to a defined stack. +-- stacktool_definebyquery_insert_random_part1.sql should be prepended. + -- Sort by the random number, and take the first N + -- to get a random set of N. + ORDER BY rnd_num + LIMIT %d +) AS randomWarps Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_part1.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_part1.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_part1.sql (revision 22235) @@ -0,0 +1,30 @@ +-- This is the first part of the query to get a list of skycells with +-- warps to be stacked, along with the number of warps already in a +-- stack. It needs to be completed by part 2 (see below). + +SELECT + skycell_id, + filter, + tess_id, + num_warp, + MAX(num_stack) AS num_stack +FROM (( + -- Number of stack-ready warps as a function of skycell and filter + SELECT + skycell_id, + warpSkyfile.tess_id as tess_id, + rawExp.filter, + COUNT(warpSkyfile.skycell_id) AS num_warp -- number of warps that can be stacked + FROM warpRun + JOIN warpSkyfile USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE + warpRun.state = 'full' + AND warpSkyfile.ignored = 0 + AND warpSkyfile.fault = 0 + -- Here should follow the SQL in stacktool_definebyquery_part2.sql, + -- after any additional WHERE conditions have been added + Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_part2.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_part2.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_part2.sql (revision 22235) @@ -0,0 +1,28 @@ + +-- This is the second part of the query to get a list of skycells with +-- warps to be stacked, along with the number of warps already in a +-- stack. It should follow part 1, which is in +-- stacktool_definebyquery_part1.sql + + GROUP BY + skycell_id, + filter + ) AS warpsToStack + LEFT JOIN ( + -- Number of stack inputs as a function of skycell and filter + SELECT + skycell_id, + stackRun.tess_id as stack_tess_id, + filter, + COUNT(stackInputSkyfile.warp_id) as num_stack -- number of warps in a stack + FROM stackRun + JOIN stackInputSkyfile USING(stack_id) + GROUP BY + stack_id + ) AS stackSizes + -- JOINing the warpsToStack and stackSizes tables + USING(skycell_id, filter) + ) + GROUP BY + skycell_id, + filter Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_test.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_test.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_definebyquery_test.sql (revision 22235) @@ -0,0 +1,59 @@ +-- This is the combination of the two parts of the query to get a list +-- of skycells with warps to be stacked, along with the number of +-- warps already in a stack. + +-------------------------------------------------------------------------------- +-- THIS FILE IS INTENDED FOR TESTING ONLY! IT IS NOT USED BY stacktool. +-- CHANGES SHOULD BE MADE TO stacktool_definebyquery_part1.sql +-- AND stacktool_definebyquery_part2.sql +-------------------------------------------------------------------------------- + +-- stacktool_definebyquery_part1.sql +SELECT + skycell_id, + tess_id, + filter, + num_warp, + MAX(num_stack) AS num_stack +FROM (( + -- Number of stack-ready warps as a function of skycell and filter + SELECT + skycell_id, + tess_id, + filter, + COUNT(warpSkyfile.skycell_id) AS num_warp -- number of warps that can be stacked + FROM warpRun + JOIN warpSkyfile USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE + warpRun.state = 'full' + AND warpSkyfile.ignored = 0 + AND warpSkyfile.fault = 0 + -- Any additional selection on warps/exposures goes here +-- stacktool_definebyquery_part2.sql + GROUP BY + skycell_id, + filter + ) AS warpsToStack + LEFT JOIN ( + -- Number of stack inputs as a function of skycell and filter + SELECT + skycell_id, + tess_id, + filter, + COUNT(stackInputSkyfile.warp_id) as num_stack -- number of warps in a stack + FROM stackRun + JOIN stackInputSkyfile USING(stack_id) + GROUP BY + stack_id + ) AS stackSizes + -- JOINing the warpsToStack and stackSizes tables + USING(skycell_id, filter) + ) + GROUP BY + skycell_id, + filter +; Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_donecleanup.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_donecleanup.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_donecleanup.sql (revision 22235) @@ -0,0 +1,5 @@ +SELECT + stackRun.* +FROM stackRun +WHERE + stackRun.state = 'cleaned' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_find_complete_warps.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_find_complete_warps.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_find_complete_warps.sql (revision 22235) @@ -0,0 +1,45 @@ +SELECT + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + stackRun.stack_id, + COUNT(warpSkyfile.skycell_id) AS num_avail, + COUNT(stackRun.stack_id) AS num_extant +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN stackInputSkyfile + ON warpSkyfile.warp_id = stackInputSkyfile.warp_id +LEFT JOIN stackRun + ON stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.stack_id = stackInputSkyfile.stack_id +WHERE + warpRun.state = 'full' + AND warpSkyfile.ignored = 0 +GROUP BY + warpSkyfile.skycell_id, stackRun.stack_id +HAVING + num_avail > num_extant + +-- It seems like we should be grouping the results here but in fact we don't +-- want to do that as each warp_id may contain multiple skycells. So a warp_id +-- may end up in more than one stackRun (for different skycells) +-- GROUP BY +-- warp_id + + +INSERT INTO + stackInputSkyfile(stack_id, warp_id) +SELECT + 13, + warp_id +FROM warpSkyfile +WHERE + skycell_id = 'new'; Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_inputskyfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_inputskyfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_inputskyfile.sql (revision 22235) @@ -0,0 +1,22 @@ +SELECT DISTINCT + warpSkyfile.*, + rawExp.camera +FROM stackRun +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON stackInputSkyfile.warp_id = warpRun.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipProcessedImfile.exp_id = rawExp.exp_id Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_pendingcleanuprun.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_pendingcleanuprun.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_pendingcleanuprun.sql (revision 22235) @@ -0,0 +1,29 @@ +-- results in too many runs adding DISTINCT fixes this +-- is the camera needed for this? YES I think so that update can define the camera +-- is the camera unique for this? YES. Well, sort of. +-- stack_skycell.pl insures that the files +-- we're here to clean come from the same camera so +-- if the camera is not unique we have nothing to do +SELECT DISTINCT + stackRun.stack_id, + rawExp.camera, + stackRun.state +FROM stackRun +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING (exp_id) +WHERE + stackRun.state = 'goto_cleaned' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_pendingcleanupskyfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_pendingcleanupskyfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_pendingcleanupskyfile.sql (revision 22235) @@ -0,0 +1,11 @@ +SELECT + stackSumSkyfile.*, + stackRun.state, + stackRun.workdir, + stackRun.dvodb, + stackRun.tess_id +FROM stackRun +JOIN stackSumSkyfile + USING(stack_id) +WHERE + stackRun.state = 'goto_cleaned' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_revertsumskyfile_delete.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_revertsumskyfile_delete.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_revertsumskyfile_delete.sql (revision 22235) @@ -0,0 +1,4 @@ +DELETE FROM stackSumSkyfile +USING stackSumSkyfile, stackRun +WHERE stackSumSkyfile.stack_id = stackRun.stack_id + AND fault != 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_revertsumskyfile_update.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_revertsumskyfile_update.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_revertsumskyfile_update.sql (revision 22235) @@ -0,0 +1,5 @@ +UPDATE stackRun +JOIN stackSumSkyfile USING(stack_id) +SET stackRun.state = 'new' +WHERE + fault != 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_sumskyfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_sumskyfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_sumskyfile.sql (revision 22235) @@ -0,0 +1,15 @@ +SELECT + stackSumSkyfile.*, + stackRun.state, + (SELECT rawExp.camera FROM + stackInputSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun ON warpRun.fake_id = fakeRun.fake_id + JOIN camRun ON camRun.cam_id = fakeRun.cam_id + JOIN chipRun ON camRun.chip_id = chipRun.chip_id + JOIN rawExp ON chipRun.exp_id = rawExp.exp_id + where stack_id = stackRun.stack_id limit 1 + ) as camera +FROM stackRun +JOIN stackSumSkyfile +USING(stack_id) Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_tosum.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_tosum.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/stacktool_tosum.sql (revision 22235) @@ -0,0 +1,18 @@ +SELECT DISTINCT + stackRun.stack_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.workdir, + stackRun.label, + stackRun.state +FROM stackRun +JOIN stackInputSkyfile + USING(stack_id) +LEFT JOIN stackSumSkyfile + USING(stack_id) +WHERE + ((stackRun.state = 'new' + AND stackSumSkyfile.stack_id IS NULL) + OR + (stackRun.state = 'update' + AND stackSumSkyfile.fault = 0)) Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_change_run_state.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_change_run_state.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_change_run_state.sql (revision 22235) @@ -0,0 +1,15 @@ +-- change state of warpRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the consituant skyfiles are in the end state +-- arguments are new state (cleaned or purged) warp_id and new state again for +-- the chipProcessedImfile sub query +UPDATE warpRun + SET state = '%s' + WHERE + warpRun.warp_id = %lld + AND (SELECT + COUNT(warp_id) + FROM warpSkyfile + WHERE + warpSkyfile.warp_id = warpRun.warp_id + AND data_state != '%s' + ) = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_change_skyfile_data_state.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_change_skyfile_data_state.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_change_skyfile_data_state.sql (revision 22235) @@ -0,0 +1,15 @@ +-- handle changes in warpSkyfile.data_state. +-- Used for the modes tocleanedskyfile and topurgedskyfile +-- args are new data_state, warp_id, skycell_id and current expected state for warpRun +UPDATE warpSkyfile + SET + data_state = '%s' +WHERE + warp_id = %lld + AND skycell_id = '%s' + -- only update if chipRun.state has the expected value + AND ( + SELECT state from warpRun where warpRun.warp_id = warpSkyfile.warp_id + ) = '%s' + + Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_definebyquery.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_definebyquery.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_definebyquery.sql (revision 22235) @@ -0,0 +1,36 @@ +SELECT + * +FROM + (SELECT DISTINCT + fakeRun.*, + chipRun.chip_id, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.exp_tag, + rawExp.exp_type, + rawExp.filelevel, + rawExp.filter, + rawExp.airmass, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.sat_pixel_frac, + rawExp.bg, + rawExp.bg_stdev, + rawExp.bg_mean_stdev, + rawExp.alt, + rawExp.az, + rawExp.ccd_temp, + rawExp.posang, + rawExp.object, + rawExp.sun_angle + FROM fakeRun + JOIN camRun + using(cam_id) + JOIN chipRun + using(chip_id) + JOIN rawExp + using(exp_id) + WHERE + fakeRun.state = 'full') as Foo Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_donecleanup.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_donecleanup.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_donecleanup.sql (revision 22235) @@ -0,0 +1,5 @@ +SELECT + warpRun.* +FROM warpRun +WHERE + warpRun.state = 'cleaned' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_exp.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_exp.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_exp.sql (revision 22235) @@ -0,0 +1,8 @@ +SELECT + fakeRun.* +FROM warpRun +JOIN fakeRun + USING(fake_id) +WHERE + warpRun.state = 'new' + AND fakeRun.state = 'full' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_imfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_imfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_imfile.sql (revision 22235) @@ -0,0 +1,28 @@ +SELECT DISTINCT + rawImfile.*, + warpRun.fake_id, + camRun.cam_id as cam_id, + chipProcessedImfile.uri as chip_uri, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base +FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id +JOIN rawImfile -- is there any reason not to refer back to rawimfiles? + ON chipProcessedImfile.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id + AND rawImfile.ignored = 0 +WHERE + warpRun.state = 'new' + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_pendingcleanuprun.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_pendingcleanuprun.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_pendingcleanuprun.sql (revision 22235) @@ -0,0 +1,15 @@ +SELECT + warpRun.warp_id, + rawExp.camera, + warpRun.state +FROM warpRun +JOIN fakeRun +USING (fake_id) +JOIN camRun +USING (cam_id) +JOIN chipRun +USING (chip_id) +JOIN rawExp +USING (exp_id) +WHERE + (warpRun.state = 'goto_cleaned' OR warpRun.state = 'goto_purged') Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_pendingcleanupskyfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 22235) @@ -0,0 +1,14 @@ +SELECT + warpSkyfile.*, + warpRun.state, + warpRun.workdir, + warpRun.label, + warpRun.dvodb, + warpRun.end_stage +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +WHERE + (warpRun.state = 'goto_cleaned' AND warpSkyfile.data_state = 'full') + OR + (warpRun.state = 'goto_purged' AND warpSkyfile.data_state != 'purged') Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_revertwarped.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_revertwarped.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_revertwarped.sql (revision 22235) @@ -0,0 +1,9 @@ +DELETE FROM warpSkyfile +USING warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp +WHERE + warpSkyfile.warp_id = warpRun.warp_id + AND warpRun.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND warpSkyfile.fault != 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_scmap.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_scmap.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_scmap.sql (revision 22235) @@ -0,0 +1,24 @@ +SELECT + warpSkyCellMap.*, + chipProcessedImfile.uri, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base +FROM warpRun +JOIN warpSkyCellMap + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND warpSkyCellMap.class_id = chipProcessedImfile.class_id +WHERE +-- warpRun.state = 'new' + fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_tooverlap.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_tooverlap.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_tooverlap.sql (revision 22235) @@ -0,0 +1,30 @@ +SELECT + warpRun.warp_id, + warpRun.fake_id, + warpRun.workdir, + warpRun.tess_id, + warpRun.label, + rawExp.camera, + exp_id, + exp_tag, + warpRun.magiced +FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN warpSkyCellMap + USING(warp_id) +LEFT JOIN warpMask + ON warpRun.label = warpMask.label +WHERE + warpRun.state = 'new' + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND warpSkyCellMap.warp_id IS NULL + AND warpMask.label IS NULL Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_towarped.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_towarped.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_towarped.sql (revision 22235) @@ -0,0 +1,42 @@ +SELECT DISTINCT + warpSkyCellMap.warp_id, + warpSkyCellMap.skycell_id, + warpSkyCellMap.tess_id, + warpRun.fake_id, + warpRun.state, + camRun.cam_id, + rawExp.camera, + rawExp.exp_tag, + warpRun.workdir +FROM warpRun +JOIN warpSkyCellMap + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN warpSkyfile + ON warpRun.warp_id = warpSkyfile.warp_id + AND warpSkyCellMap.skycell_id = warpSkyfile.skycell_id + AND warpSkyCellMap.tess_id = warpSkyfile.tess_id +LEFT JOIN warpMask + ON warpRun.label = warpMask.label +WHERE + ((warpRun.state = 'new' + AND warpSkyfile.warp_id IS NULL + AND warpSkyfile.skycell_id IS NULL + AND warpSkyfile.tess_id IS NULL) + OR (warpRun.state = 'update' + AND warpSkyfile.data_state = 'cleaned') + ) + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND warpMask.label IS NULL + AND warpSkyCellMap.fault = 0 Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_updateskyfile.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_updateskyfile.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_updateskyfile.sql (revision 22235) @@ -0,0 +1,1 @@ +UPDATE warpSkyfile SET fault = %d WHERE warp_id = %lld AND skycell_id = '%s' Index: /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_warped.sql =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_warped.sql (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/share/warptool_warped.sql (revision 22235) @@ -0,0 +1,21 @@ +SELECT + warpSkyfile.*, + warpRun.state, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +JOIN fakeRun + ON warpRun.fake_id = fakeRun.fake_id +JOIN camRun + ON camRun.cam_id = fakeRun.cam_id +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +WHERE +-- bogus test; just here so there there is a 'WHERE' stmt to append conditionals too +-- XXX EAM : not needed: fix warptool.c + warpRun.warp_id is NOT NULL Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/.cvsignore =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/.cvsignore (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/.cvsignore (revision 22235) @@ -0,0 +1,32 @@ +.deps +.gdb_history +.libs +Makefile +Makefile.in +config.h +config.h.in +stamp-h1 +*.la +*.lo +pxtoolsErrorCodes.c +pxtoolsErrorCodes.h +pxadmin +pxinject +pztool +pzgetexp +pzgetimfiles +regtool +guidetool +chiptool +camtool +warptool +difftool +stacktool +faketool +dettool +detselect +pxdata.c +magictool +caltool +flatcorr +pstamptool Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/Makefile.am =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/Makefile.am (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/Makefile.am (revision 22235) @@ -0,0 +1,220 @@ +bin_PROGRAMS = \ + caltool \ + camtool \ + chiptool \ + detselect \ + dettool \ + difftool \ + flatcorr \ + magictool \ + pstamptool \ + pxadmin \ + pxinject \ + faketool \ + pzgetexp \ + pzgetimfiles \ + pztool \ + regtool \ + stacktool \ + warptool + + +bin_SCRIPTS = \ + fakemagic + +pkginclude_HEADERS = \ + pxadmin.h \ + pxcam.h \ + pxchip.h \ + pxconfig.h \ + pxdata.h \ + pxfake.h \ + pxregister.h \ + pxtag.h \ + pxtools.h \ + pxtoolsErrorCodes.h \ + pxtree.h \ + pxwarp.h + +noinst_HEADERS = \ + caltool.h \ + camtool.h \ + chiptool.h \ + detselect.h \ + dettool.h \ + difftool.h \ + flatcorr.h \ + faketool.h \ + magictool.h \ + pstamptool.h \ + pxinject.h \ + pzgetexp.h \ + pzgetimfiles.h \ + pztool.h \ + regtool.h \ + stacktool.h \ + warptool.h + +lib_LTLIBRARIES = libpxtools.la +libpxtools_la_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +libpxtools_la_LIBS = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) +libpxtools_la_LDFLAGS = -release $(PACKAGE_VERSION) +libpxtools_la_SOURCES = \ + pxcam.c \ + pxchip.c \ + pxconfig.c \ + pxdata.c \ + pxerrors.c \ + pxfake.c \ + pxfault.c \ + pxregister.c \ + pxtag.c \ + pxtools.c \ + pxtoolsErrorCodes.c \ + pxtree.c \ + pxwarp.c + +# for pxtools.h +AM_CPPFLAGS = -I$(top_srcdir)/src$ + +pztool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pztool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pztool_SOURCES = \ + pztool.c \ + pztoolConfig.c + +pstamptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pstamptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pstamptool_SOURCES = \ + pstamptool.c \ + pstamptoolConfig.c + +caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +caltool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +caltool_SOURCES = \ + caltool.c \ + caltoolConfig.c + +flatcorr_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +flatcorr_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +flatcorr_SOURCES = \ + flatcorr.c \ + flatcorrConfig.c + +regtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +regtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +regtool_SOURCES = \ + regtool.c \ + regtoolConfig.c + +chiptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +chiptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +chiptool_SOURCES = \ + chiptool.c \ + chiptoolConfig.c + +camtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +camtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +camtool_SOURCES = \ + camtool.c \ + camtoolConfig.c + +faketool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +faketool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +faketool_SOURCES = \ + faketool.c \ + faketoolConfig.c + +magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +magictool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +magictool_SOURCES = \ + magictool.c \ + magictoolConfig.c + +warptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +warptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +warptool_SOURCES = \ + warptool.c \ + warptoolConfig.c + +difftool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +difftool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +difftool_SOURCES = \ + difftool.c \ + difftoolConfig.c + +stacktool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +stacktool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +stacktool_SOURCES = \ + stacktool.c \ + stacktoolConfig.c + +pxadmin_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pxadmin_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pxadmin_SOURCES = \ + pxadmin.c \ + pxadminConfig.c + +pzgetexp_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pzgetexp_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pzgetexp_SOURCES = \ + pzgetexp.c \ + pzgetexpConfig.c + +pzgetimfiles_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pzgetimfiles_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pzgetimfiles_SOURCES = \ + pzgetimfiles.c \ + pzgetimfilesConfig.c + +dettool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +dettool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +dettool_SOURCES = \ + dettool.c \ + dettoolConfig.c \ + dettool_correction.c \ + dettool_detrunsummary.c \ + dettool_normalizedexp.c \ + dettool_normalizedimfile.c \ + dettool_normalizedstat.c \ + dettool_processedexp.c \ + dettool_processedimfile.c \ + dettool_residexp.c \ + dettool_residimfile.c \ + dettool_stack.c + +detselect_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +detselect_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +detselect_SOURCES = \ + detselect.c \ + detselectConfig.c + +pxinject_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pxinject_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pxinject_SOURCES = \ + pxinject.c \ + pxinjectConfig.c + +clean-local: + -rm -f TAGS +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` + +# Error codes. +BUILT_SOURCES = pxtoolsErrorCodes.h pxtoolsErrorCodes.c +CLEANFILES = pxtoolsErrorCodes.h pxtoolsErrorCodes.c +EXTRA_DIST = pxtoolsErrorCodes.dat pxtoolsErrorCodes.c.in pxtoolsErrorCodes.h.in fakemagic + +pxtoolsErrorCodes.h : pxtoolsErrorCodes.dat pxtoolsErrorCodes.h.in + $(ERRORCODES) --data=pxtoolsErrorCodes.dat --outdir=. pxtoolsErrorCodes.h + +pxtoolsErrorCodes.c : pxtoolsErrorCodes.dat pxtoolsErrorCodes.c.in pxtoolsErrorCodes.h + $(ERRORCODES) --data=pxtoolsErrorCodes.dat --outdir=. pxtoolsErrorCodes.c + +BUILT_SOURCES += pxdata.c +EXTRA_DIST += pxdata.c.template + +pxdata.c: $(srcdir)/pxdata.c.template $(srcdir)/Makefile + $(PERL) -pe 's|\@PKGDATADIR\@|$(pkgdatadir)|' $< > $@ + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/caltool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/caltool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/caltool.c (revision 22235) @@ -0,0 +1,261 @@ +/* + * caltool.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include + +#include "pxtools.h" +#include "caltool.h" + +static bool adddbMode(pxConfig *config); +static bool dbsMode(pxConfig *config); +static bool addrunMode(pxConfig *config); +static bool runsMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = caltoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(CALTOOL_MODE_ADDDB, adddbMode); + MODECASE(CALTOOL_MODE_DBS, dbsMode); + MODECASE(CALTOOL_MODE_ADDRUN, addrunMode); + MODECASE(CALTOOL_MODE_RUNS, runsMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool adddbMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", true, false); + + if (!calDBInsert(config->dbh, + 0, // cal_id + dvodb, + "active" // state + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + + } + + return false; +} + + +static bool dbsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM calDB"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("caltool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "calDB", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(cal_id, config->args, "-cal_id", true, false); + PXOPT_LOOKUP_STR(region, config->args, "-region", true, false); + PXOPT_LOOKUP_STR(last_step, config->args, "-last_step", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (!calRunInsert(config->dbh, + (psS64)atoll(cal_id), + region, + last_step, + state + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + + } + + return false; +} + + +static bool runsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-cal_id", "cal_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM calRun"); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "WHERE fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "WHERE fault = 0"); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " AND %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("caltool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "calRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/caltool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/caltool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/caltool.h (revision 22235) @@ -0,0 +1,35 @@ +/* + * caltool.h + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef CALTOOL_H +#define CALTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + CALTOOL_MODE_NONE = PXTOOL_MODE_NONE, + CALTOOL_MODE_DBS, + CALTOOL_MODE_ADDDB, + CALTOOL_MODE_ADDRUN, + CALTOOL_MODE_RUNS +} caltoolMode; + +pxConfig *caltoolConfig(pxConfig *config, int argc, char **argv); + +#endif // CALTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/caltoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/caltoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/caltoolConfig.c (revision 22235) @@ -0,0 +1,104 @@ +/* + * caltoolConfig.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "caltool.h" + +pxConfig *caltoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); + psString now = psTimeToISO(time); + psFree(time); + + // -adddb + psMetadata *adddbArgs = psMetadataAlloc(); + psMetadataAddStr(adddbArgs, PS_LIST_TAIL, "-dvodb", 0, "define DVO db (required)", NULL); + + // -dbs + psMetadata *dbsArgs = psMetadataAlloc(); + psMetadataAddU64(dbsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(dbsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addrun + psMetadata *addrunArgs = psMetadataAlloc(); + psMetadataAddStr(addrunArgs, PS_LIST_TAIL, "-cal_id", 0, "define calibration DB ID (required)", NULL); + psMetadataAddStr(addrunArgs, PS_LIST_TAIL, "-region", 0, "define region (required)", NULL); + psMetadataAddStr(addrunArgs, PS_LIST_TAIL, "-last_step", 0, "define last step (required)", NULL); + psMetadataAddStr(addrunArgs, PS_LIST_TAIL, "-state", 0, "define state (required)", NULL); + + // -runs + psMetadata *runsArgs = psMetadataAlloc(); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-cal_id", 0, "search for calibration ID", NULL); + psMetadataAddU64(runsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runsArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(runsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-adddb", "add a DVO calibration DB", CALTOOL_MODE_ADDDB, adddbArgs); + PXOPT_ADD_MODE("-dbs", "list DVO calibration DBs", CALTOOL_MODE_DBS, dbsArgs); + PXOPT_ADD_MODE("-addrun", "add the results of a calibration run", CALTOOL_MODE_ADDRUN, addrunArgs); + PXOPT_ADD_MODE("-runs", "list the results of calibration runs", CALTOOL_MODE_RUNS, runsArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/camtool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/camtool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/camtool.c (revision 22235) @@ -0,0 +1,1080 @@ +/* + * camtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxcam.h" +#include "camtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupexpMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = camtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(CAMTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(CAMTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(CAMTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(CAMTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(CAMTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(CAMTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(CAMTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(CAMTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(CAMTOOL_MODE_BLOCK, blockMode); + MODECASE(CAMTOOL_MODE_MASKED, maskedMode); + MODECASE(CAMTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(CAMTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(CAMTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpMode); + MODECASE(CAMTOOL_MODE_DONECLEANUP, donecleanupMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("camtool_find_chip_id.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + // loop over our list of chipRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + chipRunRow *row = chipRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxcamQueueByChipID(config, + row->chip_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + reduction ? reduction : row->reduction, + expgroup ? expgroup : row->expgroup, + dvodb ? dvodb : row->dvodb, + tess_id ? tess_id : row->tess_id, + end_stage ? end_stage : row->end_stage + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->chip_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "camRun.state", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set chipRun.state to state + if (!pxcamRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set chipRun.label to label + if (!pxcamRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("camtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camPendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("camtool_find_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + // optional + PXOPT_LOOKUP_F32(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F32(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F32(bias, config->args, "-bias", false, false); + PXOPT_LOOKUP_F32(bias_stdev, config->args, "-bias_stdev", false, false); + PXOPT_LOOKUP_F32(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F32(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F32(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); + PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); + PXOPT_LOOKUP_F32(ap_resid, config->args, "-ap_resid", false, false); + PXOPT_LOOKUP_F32(ap_resid_stdev, config->args, "-ap_resid_stdev", false, false); + + PXOPT_LOOKUP_F32(zpt_obs, config->args, "-zpt_obs", false, false); + PXOPT_LOOKUP_F32(zpt_err, config->args, "-zpt_err", false, false); + PXOPT_LOOKUP_F32(zpt_uq, config->args, "-zpt_uq", false, false); + PXOPT_LOOKUP_F32(zpt_lq, config->args, "-zpt_lq", false, false); + + PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); + PXOPT_LOOKUP_F32(fwhm_major_lq, config->args, "-fwhm_major_lq", false, false); + PXOPT_LOOKUP_F32(fwhm_major_uq, config->args, "-fwhm_major_uq", false, false); + + PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); + PXOPT_LOOKUP_F32(fwhm_minor_lq, config->args, "-fwhm_minor_lq", false, false); + PXOPT_LOOKUP_F32(fwhm_minor_uq, config->args, "-fwhm_minor_uq", false, false); + + PXOPT_LOOKUP_F32(iq_fwhm_major, config->args, "-iq_fwhm_major", false, false); + PXOPT_LOOKUP_F32(iq_fwhm_major_err, config->args, "-iq_fwhm_major_err", false, false); + PXOPT_LOOKUP_F32(iq_fwhm_minor, config->args, "-iq_fwhm_minor", false, false); + PXOPT_LOOKUP_F32(iq_fwhm_minor_err, config->args, "-iq_fwhm_minor_err", false, false); + + PXOPT_LOOKUP_F32(iq_m2, config->args, "-iq_m2", false, false); + PXOPT_LOOKUP_F32(iq_m2_err, config->args, "-iq_m2_err", false, false); + PXOPT_LOOKUP_F32(iq_m2_uq, config->args, "-iq_m2_uq", false, false); + PXOPT_LOOKUP_F32(iq_m2_lq, config->args, "-iq_m2_lq", false, false); + + PXOPT_LOOKUP_F32(iq_m2c, config->args, "-iq_m2c", false, false); + PXOPT_LOOKUP_F32(iq_m2c_err, config->args, "-iq_m2c_err", false, false); + PXOPT_LOOKUP_F32(iq_m2c_uq, config->args, "-iq_m2c_uq", false, false); + PXOPT_LOOKUP_F32(iq_m2c_lq, config->args, "-iq_m2c_lq", false, false); + + PXOPT_LOOKUP_F32(iq_m2s, config->args, "-iq_m2s", false, false); + PXOPT_LOOKUP_F32(iq_m2s_err, config->args, "-iq_m2s_err", false, false); + PXOPT_LOOKUP_F32(iq_m2s_uq, config->args, "-iq_m2s_uq", false, false); + PXOPT_LOOKUP_F32(iq_m2s_lq, config->args, "-iq_m2s_lq", false, false); + + PXOPT_LOOKUP_F32(iq_m3, config->args, "-iq_m3", false, false); + PXOPT_LOOKUP_F32(iq_m3_err, config->args, "-iq_m3_err", false, false); + PXOPT_LOOKUP_F32(iq_m3_uq, config->args, "-iq_m3_uq", false, false); + PXOPT_LOOKUP_F32(iq_m3_lq, config->args, "-iq_m3_lq", false, false); + + PXOPT_LOOKUP_F32(iq_m4, config->args, "-iq_m4", false, false); + PXOPT_LOOKUP_F32(iq_m4_err, config->args, "-iq_m4_err", false, false); + PXOPT_LOOKUP_F32(iq_m4_uq, config->args, "-iq_m4_uq", false, false); + PXOPT_LOOKUP_F32(iq_m4_lq, config->args, "-iq_m4_lq", false, false); + + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); + PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); + PXOPT_LOOKUP_S32(n_psfstars, config->args, "-n_psfstars", false, false); + PXOPT_LOOKUP_S32(n_iqstars, config->args, "-n_iqstars", false, false); + PXOPT_LOOKUP_S32(n_extended, config->args, "-n_extended", false, false); + PXOPT_LOOKUP_S32(n_cr, config->args, "-n_cr", false, false); + PXOPT_LOOKUP_S32(n_astrom, config->args, "-n_astrom", false, false); + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + + psString query = pxDataGet("camtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClaus); + psFree(whereClaus); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]); + psFree(output); + camProcessedExpRow *row = camProcessedExpRowAlloc( + pendingRow->cam_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + bias, + bias_stdev, + fringe_0, + fringe_1, + fringe_2, + sigma_ra, + sigma_dec, + ap_resid, + ap_resid_stdev, + zpt_obs, + zpt_err, + zpt_lq, + zpt_uq, + fwhm_major, + fwhm_major_lq, + fwhm_major_uq, + fwhm_minor, + fwhm_minor_lq, + fwhm_minor_uq, + + iq_fwhm_major, + iq_fwhm_major_err, + iq_fwhm_minor, + iq_fwhm_minor_err, + + iq_m2, + iq_m2_err, + iq_m2_lq, + iq_m2_uq, + iq_m2c, + iq_m2c_err, + iq_m2c_lq, + iq_m2c_uq, + iq_m2s, + iq_m2s_err, + iq_m2s_lq, + iq_m2s_uq, + iq_m3, + iq_m3_err, + iq_m3_lq, + iq_m3_uq, + iq_m4, + iq_m4_err, + iq_m4_lq, + iq_m4_uq, + dtime_script, + dtime_astrom, + dtime_addstar, + hostname, + n_stars, + n_psfstars, + n_iqstars, + n_extended, + n_cr, + n_astrom, + path_base, + code + ); + + if (!camProcessedExpInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + // since there is only one exp per 'new' set camRun.state = 'full' + if (!pxcamRunSetState(config, row->cam_id, "full")) { + psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id); + psFree(row); + psFree(pendingRow); + return false; + } + + // NULL for end_stage means go as far as possible + // EAM : skip here if code != 0 + // Also, we can run fake even if tess_id is not defined + if (code || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam"))) { + psFree(row); + psFree(pendingRow); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + psFree(row); + // else continue on... + + if (!pxfakeQueueByCamID(config, + pendingRow->cam_id, + pendingRow->workdir, + pendingRow->label, + pendingRow->reduction, + pendingRow->expgroup, + pendingRow->dvodb, + pendingRow->tess_id, + pendingRow->end_stage + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); + psFree(pendingRow); + return false; + } + psFree(pendingRow); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + + psString query = pxDataGet("camtool_find_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND camProcessedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND camProcessedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "camProcessedExp.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("camtool_reset_faulted_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + + { + psString query = pxDataGet("camtool_revertprocessedexp.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + if (!pxSetFaultCode(config->dbh, "camProcessedExp", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!camMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM camMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM camMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("camtool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (cam_id) { + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("camtool_pendingcleanupexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camPendingCleanupExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("camtool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/camtool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/camtool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/camtool.h (revision 22235) @@ -0,0 +1,45 @@ +/* + * camtool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef CAMTOOL_H +#define CAMTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + CAMTOOL_MODE_NONE = 0x0, + CAMTOOL_MODE_DEFINEBYQUERY, + CAMTOOL_MODE_UPDATERUN, + CAMTOOL_MODE_PENDINGEXP, + CAMTOOL_MODE_PENDINGIMFILE, + CAMTOOL_MODE_ADDPROCESSEDEXP, + CAMTOOL_MODE_PROCESSEDEXP, + CAMTOOL_MODE_REVERTPROCESSEDEXP, + CAMTOOL_MODE_UPDATEPROCESSEDEXP, + CAMTOOL_MODE_BLOCK, + CAMTOOL_MODE_MASKED, + CAMTOOL_MODE_UNBLOCK, + CAMTOOL_MODE_PENDINGCLEANUPRUN, + CAMTOOL_MODE_PENDINGCLEANUPEXP, + CAMTOOL_MODE_DONECLEANUP, +} camtoolMode; + +pxConfig *camtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // CAMTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/camtoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/camtoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/camtoolConfig.c (revision 22235) @@ -0,0 +1,268 @@ +/* + * camtoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxcam.h" +#include "camtool.h" + +pxConfig *camtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *definebyqueryArgs = psMetadataAlloc(); + pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tess ID", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updaterun + // XXX need to allow multiple cam_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + pxcamSetSearchArgs(updaterunArgs); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search for state", NULL); + + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + pxcamSetSearchArgs(pendingexpArgs); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + pxcamSetSearchArgs(pendingimfileArgs); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "define camtool ID (required)", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-uri", 0, "define URI (required)", NULL); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-bias", 0, "define bias", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-bias_stdev", 0, "define bias stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe term 0", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe term 1", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe term 2", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-sigma_ra", 0, "define exposure E ra", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-sigma_dec", 0, "define exposure E dec", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ap_resid", 0, "define aperture residual", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ap_resid_stdev", 0, "define aperture residual stdev", NAN); + + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-zpt_obs", 0, "define observed zero point", 0); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-zpt_err", 0, "define observed zero point error", 0); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-zpt_uq", 0, "define observed zero point upper quartile", 0); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-zpt_lq", 0, "define observed zero point lower quartile", 0); + + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_major", 0, "define fwhm (major axis; arcsec)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_major_lq", 0, "define fwhm (major axis; arcsec)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_major_uq", 0, "define fwhm (major axis; arcsec)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_minor", 0, "define fwhm (minor axis; arcsec)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_minor_lq", 0, "define fwhm (minor axis; arcsec)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_minor_uq", 0, "define fwhm (minor axis; arcsec)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_fwhm_major", 0, "define moment value m2 mean", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_fwhm_major_err", 0, "define moment value m2 mean", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_fwhm_minor", 0, "define moment value m2 mean", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_fwhm_minor_err", 0, "define moment value m2 mean", NAN); + + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2", 0, "define moment value m2 mean", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2_err", 0, "define moment value m2 stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2_lq", 0, "define moment value m2 lower quartile", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2_uq", 0, "define moment value m2 upper quartile", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2c", 0, "define moment value m2c mean", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2c_err", 0, "define moment value m2c stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2c_lq", 0, "define moment value m2c lower quartile", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2c_uq", 0, "define moment value m2c upper quartile", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2s", 0, "define moment value m2s mean", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2s_err", 0, "define moment value m2s stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2s_lq", 0, "define moment value m2s lower quartile", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m2s_uq", 0, "define moment value m2s upper quartile", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m3", 0, "define moment value m3 mean", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m3_err", 0, "define moment value m3 stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m3_lq", 0, "define moment value m3 lower quartile", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m3_uq", 0, "define moment value m3 upper quartile", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m4", 0, "define moment value m4 mean", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m4_err", 0, "define moment value m4 stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m4_lq", 0, "define moment value m4 lower quartile", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-iq_m4_uq", 0, "define moment value m4 upper quartile", NAN); + + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_astrom", 0, "define elapsed time for astrometry (seconds)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_psfstars", 0, "define number of PSF stars", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_iqstars", 0, "define number of IQ stars", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_extended", 0, "define number of extended objects", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_cr", 0, "define number of cosmic rays", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_astrom", 0, "define number of astrometry reference objects", 0); + + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddBool(addprocessedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + pxcamSetSearchArgs(processedexpArgs); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + + // -revertprocessedexp + // XXX need to allow multiple cam_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + pxcamSetSearchArgs(revertprocessedexpArgs); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + + psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateprocessedexp + // XXX allow full search options? + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chiptool ID", 0); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", INT16_MAX); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + // XXX allow full search options? + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupexp + // XXX allow full search options? + psMetadata *pendingcleanupexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camera ID", 0); + psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", NULL); + psMetadataAddBool(pendingcleanupexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from chip stage", CAMTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change cam run properties", CAMTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exposures", CAMTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-pendingimfile", "show pending imfiles", CAMTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-addprocessedexp", "add a processed exposure", CAMTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "show processed exposures", CAMTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "change procesed exp properties", CAMTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "undo a processed exposure", CAMTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXOPT_ADD_MODE("-block", "set a label block", CAMTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", CAMTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", CAMTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", CAMTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupexp", "show exposures for cleanup runs", CAMTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", CAMTOOL_MODE_DONECLEANUP, donecleanupArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/chiptool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/chiptool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/chiptool.c (revision 22235) @@ -0,0 +1,1216 @@ +/* + * chiptool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "pxchip.h" + +#include "chiptool.h" +#include "camtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool revertprocessedimfileMode(pxConfig *config); +static bool updateprocessedimfileMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unmaskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupimfileMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool runMode(pxConfig *config); +static bool tocleanedimfileMode(pxConfig *config); +static bool tofullimfileMode(pxConfig *config); +static bool topurgedimfileMode(pxConfig *config); + +static bool chipProcessedCompleteExp(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = chiptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(CHIPTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(CHIPTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(CHIPTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); + MODECASE(CHIPTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(CHIPTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); + MODECASE(CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + MODECASE(CHIPTOOL_MODE_BLOCK, blockMode); + MODECASE(CHIPTOOL_MODE_MASKED, maskedMode); + MODECASE(CHIPTOOL_MODE_UNMASKED, unmaskedMode); + MODECASE(CHIPTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileMode); + MODECASE(CHIPTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(CHIPTOOL_MODE_RUN, runMode); + MODECASE(CHIPTOOL_MODE_TOCLEANEDIMFILE, tocleanedimfileMode); + MODECASE(CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileMode); + MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp only + PXOPT_COPY_STR(config->args, where, "-label", "rawExp.label", "LIKE"); + + // psListLength(where->list) is at least 1 because exp_type defaults to "object" + // so we require a list longer than 1 entry + if ((psListLength(where->list) <= 1) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + + // default + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("chiptool_find_rawexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of exp_ids + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psFree(output); + return false; + } + + // queue the exp + if (!pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue exp_id: %" PRId64, exp_id); + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp, chipRun + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + where = NULL; + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set chipRun.state to state + if (!pxchipRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set chipRun.label to label + if (!pxchipRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); //chipRun, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); + + psString query = pxDataGet("chiptool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // chip_id, exp_tag, class_id are required + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_F32(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F32(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F32(bias, config->args, "-bias", false, false); + PXOPT_LOOKUP_F32(bias_stdev, config->args, "-bias_stdev", false, false); + PXOPT_LOOKUP_F32(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F32(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F32(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F32(ap_resid, config->args, "-ap_resid", false, false); + PXOPT_LOOKUP_F32(ap_resid_stdev, config->args, "-ap_resid_stdev", false, false); + + PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); + PXOPT_LOOKUP_F32(fwhm_major_lq, config->args, "-fwhm_major_lq", false, false); + PXOPT_LOOKUP_F32(fwhm_major_uq, config->args, "-fwhm_major_uq", false, false); + + PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); + PXOPT_LOOKUP_F32(fwhm_minor_lq, config->args, "-fwhm_minor_lq", false, false); + PXOPT_LOOKUP_F32(fwhm_minor_uq, config->args, "-fwhm_minor_uq", false, false); + + PXOPT_LOOKUP_F32(iq_fwhm_major, config->args, "-iq_fwhm_major", false, false); + PXOPT_LOOKUP_F32(iq_fwhm_major_err, config->args, "-iq_fwhm_major_err", false, false); + PXOPT_LOOKUP_F32(iq_fwhm_minor, config->args, "-iq_fwhm_minor", false, false); + PXOPT_LOOKUP_F32(iq_fwhm_minor_err, config->args, "-iq_fwhm_minor_err", false, false); + + PXOPT_LOOKUP_F32(iq_m2, config->args, "-iq_m2", false, false); + PXOPT_LOOKUP_F32(iq_m2_err, config->args, "-iq_m2_err", false, false); + PXOPT_LOOKUP_F32(iq_m2_uq, config->args, "-iq_m2_uq", false, false); + PXOPT_LOOKUP_F32(iq_m2_lq, config->args, "-iq_m2_lq", false, false); + + PXOPT_LOOKUP_F32(iq_m2c, config->args, "-iq_m2c", false, false); + PXOPT_LOOKUP_F32(iq_m2c_err, config->args, "-iq_m2c_err", false, false); + PXOPT_LOOKUP_F32(iq_m2c_uq, config->args, "-iq_m2c_uq", false, false); + PXOPT_LOOKUP_F32(iq_m2c_lq, config->args, "-iq_m2c_lq", false, false); + + PXOPT_LOOKUP_F32(iq_m2s, config->args, "-iq_m2s", false, false); + PXOPT_LOOKUP_F32(iq_m2s_err, config->args, "-iq_m2s_err", false, false); + PXOPT_LOOKUP_F32(iq_m2s_uq, config->args, "-iq_m2s_uq", false, false); + PXOPT_LOOKUP_F32(iq_m2s_lq, config->args, "-iq_m2s_lq", false, false); + + PXOPT_LOOKUP_F32(iq_m3, config->args, "-iq_m3", false, false); + PXOPT_LOOKUP_F32(iq_m3_err, config->args, "-iq_m3_err", false, false); + PXOPT_LOOKUP_F32(iq_m3_uq, config->args, "-iq_m3_uq", false, false); + PXOPT_LOOKUP_F32(iq_m3_lq, config->args, "-iq_m3_lq", false, false); + + PXOPT_LOOKUP_F32(iq_m4, config->args, "-iq_m4", false, false); + PXOPT_LOOKUP_F32(iq_m4_err, config->args, "-iq_m4_err", false, false); + PXOPT_LOOKUP_F32(iq_m4_uq, config->args, "-iq_m4_uq", false, false); + PXOPT_LOOKUP_F32(iq_m4_lq, config->args, "-iq_m4_lq", false, false); + + PXOPT_LOOKUP_F32(dtime_detrend, config->args, "-dtime_detrend", false, false); + PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); + PXOPT_LOOKUP_F32(dtime_total, config->args, "-dtime_total", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(n_stars, config->args, "-n_stars", false, false); + PXOPT_LOOKUP_F32(n_psfstars, config->args, "-n_psfstars", false, false); + PXOPT_LOOKUP_F32(n_iqstars, config->args, "-n_iqstars", false, false); + PXOPT_LOOKUP_F32(n_extended, config->args, "-n_extended", false, false); + PXOPT_LOOKUP_F32(n_cr, config->args, "-n_cr", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!chipProcessedImfileInsert(config->dbh, + chip_id, + exp_id, + class_id, + "full", + uri, + bg, + bg_stdev, + bg_mean_stdev, + bias, + bias_stdev, + fringe_0, + fringe_1, + fringe_2, + ap_resid, + ap_resid_stdev, + + fwhm_major, + fwhm_major_lq, + fwhm_major_uq, + fwhm_minor, + fwhm_minor_lq, + fwhm_minor_uq, + + iq_fwhm_major, + iq_fwhm_major_err, + iq_fwhm_minor, + iq_fwhm_minor_err, + + iq_m2, + iq_m2_err, + iq_m2_lq, + iq_m2_uq, + + iq_m2c, + iq_m2c_err, + iq_m2c_lq, + iq_m2c_uq, + + iq_m2s, + iq_m2s_err, + iq_m2s_lq, + iq_m2s_uq, + + iq_m3, + iq_m3_err, + iq_m3_lq, + iq_m3_uq, + + iq_m4, + iq_m4_err, + iq_m4_lq, + iq_m4_uq, + + dtime_detrend, + dtime_photom, + dtime_total, + dtime_script, + hostname, + n_stars, + n_psfstars, + n_iqstars, + n_extended, + n_cr, + path_base, + code + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX I've decided to make the transaction cover the Exp migration as + // well. Otherwise, if the last imfile in an exp is moved and the exp + // migration fails then the data base is left in a situation where the exp + // migration can't happen. + + if (!chipProcessedCompleteExp(config)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "LIKE"); + + psString query = pxDataGet("chiptool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "chipProcessedImfile.fault", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("chiptool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool updateprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + psFree(where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!chipMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM chipMask"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// return the list of labels which are NOT blocked +static bool unmaskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + psString query = pxDataGet("chiptool_unmasked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, "chipUnmask"); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipUnmask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM chipMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("chiptool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (chip_id) { + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("chiptool_pendingcleanupimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingCleanupImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("chiptool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool runMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // make sure that the state string is valid + if (!pxIsValidState(state)) { + psError(PXTOOLS_ERR_DATA, false, "%s is not a valid state", state); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + + psString query = pxDataGet("chiptool_run.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool chipProcessedCompleteExp(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // look for completed chipPendingExp + // migrate them to chipProccessedExp & camPendingExp + psString query = pxDataGet("chiptool_completely_processed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + chipRunRow *chipRun = chipRunObjectFromMetadata(row); + // set chipRun.state to 'stop' + if (!pxchipRunSetState(config, chipRun->chip_id, "full")) { + psError(PS_ERR_UNKNOWN, false, "failed to change chipRun.state for chip_id: %" PRId64, chipRun->chip_id); + psFree(chipRun); + psFree(output); + return false; + } + + // should we stop here or proceed on to the cam stage? + // NULL for end_stage means go as far as possible + if (chipRun->end_stage && psStrcasestr(chipRun->end_stage, "chip")) { + psFree(chipRun); + continue; + } + // else continue on... + + // camQueueChipID() can only be run after chipRun.state has been set to + // stop + if (!pxcamQueueByChipID(config, + chipRun->chip_id, + chipRun->workdir, + chipRun->label, + chipRun->reduction, + chipRun->expgroup, + chipRun->dvodb, + chipRun->tess_id, + chipRun->end_stage + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue camPendingExp"); + psFree(chipRun); + psFree(output); + return false; + } + psFree(chipRun); + } + + psFree(output); + + return true; +} + +// update chipProcessedImfile.data_state to given value. +// afterwards, if all imfiles in the exposure have the new state, update the state for the exposure as well +// shared code for the modes -tocleanedimfile -tofullimfile -topurgedimfile + +static bool change_imfile_data_state(pxConfig *config, psString data_state, psString run_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // chip_id, class_id are required + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + psString query = pxDataGet("chiptool_change_imfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // note only updates if chipRun.state = run_state + if (!p_psDBRunQuery(config->dbh, query, data_state, chip_id, class_id, run_state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + query = pxDataGet("chiptool_change_exp_state.sql"); + if (!p_psDBRunQuery(config->dbh, query, data_state, chip_id, data_state)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool tocleanedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "cleaned", "goto_cleaned"); +} +static bool tofullimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "full", "update"); +} +static bool topurgedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "purged", "goto_purged"); +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/chiptool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/chiptool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/chiptool.h (revision 22235) @@ -0,0 +1,50 @@ +/* + * chiptool.h + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef CHIPTOOL_H +#define CHIPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + CHIPTOOL_MODE_NONE = 0x0, + CHIPTOOL_MODE_DEFINEBYQUERY, + CHIPTOOL_MODE_UPDATERUN, + CHIPTOOL_MODE_PENDINGIMFILE, + CHIPTOOL_MODE_ADDPROCESSEDIMFILE, + CHIPTOOL_MODE_PROCESSEDIMFILE, + CHIPTOOL_MODE_REVERTPROCESSEDIMFILE, + CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, + CHIPTOOL_MODE_BLOCK, + CHIPTOOL_MODE_MASKED, + CHIPTOOL_MODE_UNMASKED, + CHIPTOOL_MODE_UNBLOCK, + CHIPTOOL_MODE_RETRYPROCESSEDIMFILE, + CHIPTOOL_MODE_PENDINGCLEANUPRUN, + CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, + CHIPTOOL_MODE_DONECLEANUP, + CHIPTOOL_MODE_RUN, + CHIPTOOL_MODE_TOCLEANEDIMFILE, + CHIPTOOL_MODE_TOFULLIMFILE, + CHIPTOOL_MODE_TOPURGEDIMFILE +} chiptoolMode; + +pxConfig *chiptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // CHIPTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/chiptoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/chiptoolConfig.c (revision 22235) @@ -0,0 +1,281 @@ +/* + * chiptoolConfig.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "chiptool.h" + +pxConfig *chiptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + pxchipSetSearchArgs (definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "search by rawExp label (LIKE comparison)", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_type", PS_META_REPLACE, "search by exp_type", "object"); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + pxchipSetSearchArgs (updaterunArgs); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + pxchipSetSearchArgs(pendingimfileArgs); + psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip ID (required)", 0); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID (required)", 0); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri", 0, "define URL", NULL); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bias", 0, "define bias", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bias_stdev", 0, "define bias stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe term 0", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe term 1", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe term 2", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-ap_resid", 0, "define aperture residual", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-ap_resid_stdev", 0, "define aperture residual stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fwhm_major", 0, "define fwhm (major axis; arcsec)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fwhm_major_lq", 0, "define fwhm (major axis; arcsec)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fwhm_major_uq", 0, "define fwhm (major axis; arcsec)", NAN); + + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fwhm_minor", 0, "define fwhm (minor axis; arcsec)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fwhm_minor_lq", 0, "define fwhm (minor axis; arcsec)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fwhm_minor_uq", 0, "define fwhm (minor axis; arcsec)", NAN); + + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_fwhm_major", 0, "define moment value m2 mean", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_fwhm_major_err", 0, "define moment value m2 mean", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_fwhm_minor", 0, "define moment value m2 mean", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_fwhm_minor_err", 0, "define moment value m2 mean", NAN); + + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2", 0, "define moment value m2 mean", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2_err", 0, "define moment value m2 stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2_lq", 0, "define moment value m2 lower quartile", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2_uq", 0, "define moment value m2 upper quartile", NAN); + + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2c", 0, "define moment value m2c mean", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2c_err", 0, "define moment value m2c stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2c_lq", 0, "define moment value m2c lower quartile", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2c_uq", 0, "define moment value m2c upper quartile", NAN); + + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2s", 0, "define moment value m2s mean", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2s_err", 0, "define moment value m2s stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2s_lq", 0, "define moment value m2s lower quartile", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m2s_uq", 0, "define moment value m2s upper quartile", NAN); + + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m3", 0, "define moment value m3 mean", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m3_err", 0, "define moment value m3 stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m3_lq", 0, "define moment value m3 lower quartile", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m3_uq", 0, "define moment value m3 upper quartile", NAN); + + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m4", 0, "define moment value m4 mean", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m4_err", 0, "define moment value m4 stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m4_lq", 0, "define moment value m4 lower quartile", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-iq_m4_uq", 0, "define moment value m4 upper quartile", NAN); + + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-dtime_detrend", 0, "define elapsed time for detrend (seconds)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-dtime_photom", 0, "define elapsed time for photometry (seconds)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-dtime_total", 0, "define total elapsed time (seconds)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars in image", 0); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_psfstars", 0, "define number of stars used for psf", 0); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_iqstars", 0, "define number of stars used for IQ stats", 0); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_extended", 0, "define number of extended objects in image", 0); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_cr", 0, "define number of cosmic rays in image", 0); + + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip ID", 0); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-label", 0, "search by chipRun label (LIKE comparison)", NULL); + pxchipSetSearchArgs(processedimfileArgs); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertprocessedimfile + psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-label", 0, "search by chipRun label (LIKE comparison)", NULL); + pxchipSetSearchArgs(revertprocessedimfileArgs); + psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateprocessedimfile + psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", 0); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddStr(maskedArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(maskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unmasked + psMetadata *unmaskedArgs = psMetadataAlloc(); + psMetadataAddStr(unmaskedArgs, PS_LIST_TAIL, "-label", 0, "restrict to specified label", NULL); + psMetadataAddBool(unmaskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(unmaskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupimfile + psMetadata *pendingcleanupimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", NULL); + psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -run + psMetadata *runArgs = psMetadataAlloc(); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddBool(runArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(runArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); + + // -tocleanedimfile + psMetadata *tocleanedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(tocleanedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -tofullimfile + psMetadata *tofullimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(tofullimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -topurgedimfile + psMetadata *topurgedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(topurgedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from raw exposures", CHIPTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change chip run properties", CHIPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingimfile", "show pending imfiles", CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-addprocessedimfile", "add a processed imfile", CHIPTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXOPT_ADD_MODE("-processedimfile", "show processed imfiles", CHIPTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileArgs); + PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile", CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs); + PXOPT_ADD_MODE("-block", "set a label block", CHIPTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", CHIPTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unmasked", "", CHIPTOOL_MODE_UNMASKED, unmaskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", CHIPTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", CHIPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupimfile", "show runs that need to be cleaned up", CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", CHIPTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-run", "show runs", CHIPTOOL_MODE_RUN, runArgs); + PXOPT_ADD_MODE("-tocleanedimfile", "set imfile state to cleaned", CHIPTOOL_MODE_TOCLEANEDIMFILE, tocleanedimfileArgs); + PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileArgs); + PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/detselect.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/detselect.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/detselect.c (revision 22235) @@ -0,0 +1,220 @@ +/* + * detselect.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "detselect.h" + +static bool searchMode(pxConfig *config); +static bool selectMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = detselectConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DETSELECT_MODE_SEARCH, searchMode); + MODECASE(DETSELECT_MODE_SELECT, selectMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool searchMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(unlimit, config->args, "-unlimit", false); + + PXOPT_LOOKUP_TIME(time, config->args, "-time", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + PXOPT_COPY_STR(config->args, where, "-type", "det_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + // airmass_min < airmass < airmass_max + PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_min", "<="); + PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_max", ">="); + + // exp_time_min < exp_time < exp_time_max + PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_min", "<="); + PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_max", ">="); + + // ccd_temp_min < ccd_temp < ccd_temp_max + PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_min", "<="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_max", ">="); + + PXOPT_COPY_F64(config->args, where, "-posang", "posang_min", "<="); + PXOPT_COPY_F64(config->args, where, "-posang", "posang_max", ">="); + + // time_begin < time < time_end + // the == NULL tests invokes some psDB magic to make an OR + // conditional query + psMetadataAddTime(where, PS_LIST_TAIL, "time_begin", 0, "<=", time); + psMetadataAddTime(where, PS_LIST_TAIL, "time_begin", PS_META_DUPLICATE_OK, "==", NULL); + psMetadataAddTime(where, PS_LIST_TAIL, "time_end", 0, ">=", time); + psMetadataAddTime(where, PS_LIST_TAIL, "time_end", PS_META_DUPLICATE_OK, "==", NULL); + + psString query = pxDataGet("detselect_search.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // we choose the single detrend image which matches all criteria and has + // the latest insertion date + + // unless explicitly specified by the user, list all possible matches + if (!unlimit) { + psStringAppend(&query, " ORDER BY registered DESC LIMIT 1"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detrend exposures found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool selectMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("detselect_select.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detNormalizedImfile rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/detselect.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/detselect.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/detselect.h (revision 22235) @@ -0,0 +1,33 @@ +/* + * detselect.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DETSELECT_H +#define DETSELECT_H 1 + +#include "pxtools.h" + +typedef enum { + DETSELECT_MODE_NONE = 0x0, + DETSELECT_MODE_SEARCH, + DETSELECT_MODE_SELECT, +} detselectMode; + +pxConfig *detselectConfig(pxConfig *config, int argc, char **argv); + +#endif // DETSELECT_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/detselectConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/detselectConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/detselectConfig.c (revision 22235) @@ -0,0 +1,94 @@ +/* + * detselectConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pxtools.h" +#include "detselect.h" + +pxConfig *detselectConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -search + psMetadata *searchArgs = psMetadataAlloc(); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-det_type", 0, "search by detrend type", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-type", 0, "search by detrend type (alias for -det_type)", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddF32(searchArgs, PS_LIST_TAIL, "-airmass", 0, "define airmass", NAN); + psMetadataAddF32(searchArgs, PS_LIST_TAIL, "-exp_time", 0, "search by exposure time", NAN); + psMetadataAddF32(searchArgs, PS_LIST_TAIL, "-ccd_temp", 0, "search by ccd tempature", NAN); + psMetadataAddF64(searchArgs, PS_LIST_TAIL, "-posang", 0, "search by rotator position angle", NAN); + psMetadataAddTime(searchArgs, PS_LIST_TAIL, "-time", 0, "define time for desired detrend data", NULL); + psMetadataAddBool(searchArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(searchArgs, PS_LIST_TAIL, "-unlimit", 0, "list all possible detruns, not just the best match", false); + + // -select + psMetadata *selectArgs = psMetadataAlloc(); + psMetadataAddStr(selectArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", NULL); + psMetadataAddS32(selectArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(selectArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddBool(selectArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-search", "search for an appropriate detrend", DETSELECT_MODE_SEARCH, searchArgs); + PXOPT_ADD_MODE("-select", "retreive detrend information", DETSELECT_MODE_SELECT, selectArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool.c (revision 22235) @@ -0,0 +1,2088 @@ +/* + * dettool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +static bool pendingMode(pxConfig *config); +static bool definebytagMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); +static bool definebydetrunMode(pxConfig *config); +static bool runsMode(pxConfig *config); +static bool childlessrunMode(pxConfig *config); +static bool inputMode(pxConfig *config); +static bool rawMode(pxConfig *config); + +// run +static bool updatedetrunMode(pxConfig *config); +static bool rerunMode(pxConfig *config); +// register +static bool register_detrendMode(pxConfig *config); + +//static psArray *validDetInputClassIds(pxConfig *config, const char *det_id); +//static psArray *searchInputImfiles(pxConfig *config, const char *det_id); +static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS64 det_id, psS32 iteration); +static psS32 incrementIteration(pxConfig *config, psS64 det_id); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = dettoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DETTOOL_MODE_PENDING, pendingMode); + MODECASE(DETTOOL_MODE_DEFINEBYTAG, definebytagMode); + MODECASE(DETTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DETTOOL_MODE_DEFINEBYDETRUN, definebydetrunMode); + MODECASE(DETTOOL_MODE_RUNS, runsMode); + MODECASE(DETTOOL_MODE_CHILDLESSRUN, childlessrunMode); + MODECASE(DETTOOL_MODE_INPUT, inputMode); + MODECASE(DETTOOL_MODE_RAW, rawMode); + // correction + MODECASE(DETTOOL_MODE_MAKECORRECTION, makecorrectionMode); + MODECASE(DETTOOL_MODE_TOCORRECTEXP, tocorrectexpMode); + MODECASE(DETTOOL_MODE_TOCORRECTIMFILE, tocorrectimfileMode); + MODECASE(DETTOOL_MODE_ADDCORRECTIMFILE, addcorrectimfileMode); + // imfile + MODECASE(DETTOOL_MODE_TOPROCESSEDIMFILE,toprocessedimfileMode); + MODECASE(DETTOOL_MODE_ADDPROCESSEDIMFILE,addprocessedimfileMode); + MODECASE(DETTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(DETTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); + MODECASE(DETTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDIMFILE, pendingcleanup_processedimfileMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_PROCESSEDIMFILE, donecleanup_processedimfileMode); + // exp + MODECASE(DETTOOL_MODE_TOPROCESSEDEXP, toprocessedexpMode); + MODECASE(DETTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(DETTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(DETTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(DETTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, pendingcleanup_processedexpMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, donecleanup_processedexpMode); + // stacked + MODECASE(DETTOOL_MODE_TOSTACKED, tostackedMode); + MODECASE(DETTOOL_MODE_ADDSTACKED, addstackedMode); + MODECASE(DETTOOL_MODE_STACKED, stackedMode); + MODECASE(DETTOOL_MODE_REVERTSTACKED, revertstackedMode); + MODECASE(DETTOOL_MODE_UPDATESTACKED, updatestackedMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_STACKED, pendingcleanup_stackedMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_STACKED, donecleanup_stackedMode); + // normalizedstat + MODECASE(DETTOOL_MODE_TONORMALIZEDSTAT, tonormalizedstatMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDSTAT,addnormalizedstatMode); + MODECASE(DETTOOL_MODE_NORMALIZEDSTAT, normalizedstatMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDSTAT, revertnormalizedstatMode); + MODECASE(DETTOOL_MODE_UPDATENORMALIZEDSTAT, updatenormalizedstatMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDSTAT, pendingcleanup_normalizedstatMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_NORMALIZEDSTAT, donecleanup_normalizedstatMode); + // normalizedimfile + MODECASE(DETTOOL_MODE_TONORMALIZE, tonormalizeMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDIMFILE,addnormalizedimfileMode); + MODECASE(DETTOOL_MODE_NORMALIZEDIMFILE, normalizedimfileMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDIMFILE, revertnormalizedimfileMode); + MODECASE(DETTOOL_MODE_UPDATENORMALIZEDIMFILE, updatenormalizedimfileMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDIMFILE, pendingcleanup_normalizedimfileMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_NORMALIZEDIMFILE, donecleanup_normalizedimfileMode); + // normalizedexp + MODECASE(DETTOOL_MODE_TONORMALIZEDEXP, tonormalizedexpMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDEXP, addnormalizedexpMode); + MODECASE(DETTOOL_MODE_NORMALIZEDEXP, normalizedexpMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDEXP, revertnormalizedexpMode); + MODECASE(DETTOOL_MODE_UPDATENORMALIZEDEXP, updatenormalizedexpMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDEXP, pendingcleanup_normalizedexpMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_NORMALIZEDEXP, donecleanup_normalizedexpMode); + // residimfile + MODECASE(DETTOOL_MODE_TORESIDIMFILE, toresidimfileMode); + MODECASE(DETTOOL_MODE_ADDRESIDIMFILE, addresidimfileMode); + MODECASE(DETTOOL_MODE_RESIDIMFILE, residimfileMode); + MODECASE(DETTOOL_MODE_REVERTRESIDIMFILE,revertresidimfileMode); + MODECASE(DETTOOL_MODE_UPDATERESIDIMFILE, updateresidimfileMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_RESIDIMFILE, pendingcleanup_residimfileMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_RESIDIMFILE, donecleanup_residimfileMode); + // residexp + MODECASE(DETTOOL_MODE_TORESIDEXP, toresidexpMode); + MODECASE(DETTOOL_MODE_ADDRESIDEXP, addresidexpMode); + MODECASE(DETTOOL_MODE_RESIDEXP, residexpMode); + MODECASE(DETTOOL_MODE_REVERTRESIDEXP, revertresidexpMode); + MODECASE(DETTOOL_MODE_UPDATERESIDEXP, updateresidexpMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, pendingcleanup_residexpMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_RESIDEXP, donecleanup_residexpMode); + // detrunsummary + MODECASE(DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryMode); + MODECASE(DETTOOL_MODE_ADDDETRUNSUMMARY, adddetrunsummaryMode); + MODECASE(DETTOOL_MODE_DETRUNSUMMARY, detrunsummaryMode); + MODECASE(DETTOOL_MODE_REVERTDETRUNSUMMARY, revertdetrunsummaryMode); + MODECASE(DETTOOL_MODE_UPDATEDETRUNSUMMARY, updatedetrunsummaryMode); + MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); + MODECASE(DETTOOL_MODE_RERUN, rerunMode); + // register + MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); + MODECASE(DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool pendingMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_pending.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool definebytagMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // what type of detRun is this? + // required + PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + + // optional + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", false, false); + + // check mode + PXOPT_LOOKUP_STR(mode, config->args, "-mode", true, false); // default ('master') is supplied + if (!isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalid mode"); + return false; + } + + // get -ref_det_id and -ref_iter : required for 'verify' mode / disallowed otherwise + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", true, false); + PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", true, false); + if (!strcmp(mode, "verify") && ((ref_det_id == 0) || (ref_iter == -1))) { + psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); + return false; + } + if (strcmp(mode, "verify") && ((ref_det_id != 0) || (ref_iter != -1))) { + psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); + return false; + } + + PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + + PXOPT_LOOKUP_F32(airmass_min, config->args, "-airmass_min", false, false); + PXOPT_LOOKUP_F32(airmass_max, config->args, "-airmass_max", false, false); + PXOPT_LOOKUP_F32(exp_time_min, config->args, "-exp_time_min", false, false); + PXOPT_LOOKUP_F32(exp_time_max, config->args, "-exp_time_max", false, false); + PXOPT_LOOKUP_F32(ccd_temp_min, config->args, "-ccd_temp_min", false, false); + PXOPT_LOOKUP_F32(ccd_temp_max, config->args, "-ccd_temp_max", false, false); + PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); + PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); + PXOPT_LOOKUP_TIME(time_end, config->args, "-time_end", false, false); + PXOPT_LOOKUP_TIME(use_begin, config->args, "-use_begin", false, false); + PXOPT_LOOKUP_TIME(use_end, config->args, "-use_end", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + // default + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + + // we have to support multiple exp_ids + psMetadataItem *item = psMetadataLookup(config->args, "-exp_id"); + if (!item) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "at least one -exp_id is required"); + return false; + } + psMetadata *where = psMetadataAlloc(); + + if ((item->type != PS_DATA_METADATA_MULTI) && (item->type != PS_DATA_S64)) { + psAbort("-exp_id was not parsed correctly (this should not happen"); + } + + // make sure that -exp_id was parsed correctly + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psS64 exp_id = mItem->data.S64; + if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } + if (item->type == PS_DATA_S64) { + psS64 exp_id = item->data.S64; + if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } + + if (psListLength(where->list) < 1) { + psFree(where); + where = NULL; + } + + // check that the specified exp_ids actually exist + psArray *detrendExps = rawExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return false; + } + + // we should have one rawExp row per exp_id specified + if (psListLength(item->data.list) != psArrayLength(detrendExps)) { + psAbort( "an -exp_id matched more then one rawExp (this should not happen"); + + } + + // check to see if -filelevel was set on the command line + if (!filelevel) { + filelevel = psStringCopy(((rawExpRow *)detrendExps->data[0])->filelevel); + } + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detrendExps); + return false; + } + + // the first iteration is always 0 + // XXX the camera name is set from the first inputExp + // XXX det_id + detRunInsert(config->dbh, + 0, + 0, + det_type, + mode, + "run", + filelevel, + workdir, + camera, + telescope, + exp_type, + reduction, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + ref_det_id, + ref_iter + ); + psS64 det_id = psDBLastInsertID(config->dbh); + + // create new detInputExp row(s) from the rawExp row(s) + psArray *inputExps = psArrayAllocEmpty(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + detInputExpRow *inputExp = rawDetrenTodetInputExpRow( + detrendExps->data[i], + det_id, + 0 // the first iteration is explicitly 0 + ); + psArrayAdd(inputExps, 0, inputExp); + psFree(inputExp); + } + + psFree(detrendExps); + + // insert detInputExp objects into the database + if (!detInputExpInsertObjects(config->dbh, inputExps)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(inputExps); + return false; + } + psFree(inputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // print the new det_id + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", det_id); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +#if 0 +// This function is used to convert the det_id from an int, as it is used +// internally, to be a string for external use. The rationale being that we may +// want to change how det_id is generated in the future and don't want to +// external programs to become depending on this value being an int. +static bool convertDetIdToStr(psArray *mds) +{ + PS_ASSERT_PTR_NON_NULL(mds, false); + + for (long i = 0; i < psArrayLength(mds); i++) { + psMetadata *md = mds->data[i]; + bool status = false; + psS32 det_id = psMetadataLookupS32(&status, md, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psMetadataRemoveKey(md, "det_id"); + psString det_idStr = psDBIntToString((psU64)det_id); + psMetadataAddStr(mds->data[i], PS_LIST_HEAD, "det_id", 0, NULL, det_idStr); + psFree(det_idStr); + } + + return true; +} +#endif + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + + // check mode + PXOPT_LOOKUP_STR(mode, config->args, "-mode", true, false); // default ('master') is supplied + if (!isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalid mode"); + return false; + } + + // get -ref_det_id and -ref_iter : required for 'verify' mode / disallowed otherwise + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", false, false); + PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", false, false); + if (!strcmp(mode, "verify") && ((ref_det_id == 0) || (ref_iter == -1))) { + psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); + return false; + } + if (strcmp(mode, "verify") && ((ref_det_id != 0) || (ref_iter != -1))) { + psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); + return false; + } + + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", false, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_F32(airmass_min, config->args, "-airmass_min", false, false); + PXOPT_LOOKUP_F32(airmass_max, config->args, "-airmass_max", false, false); + PXOPT_LOOKUP_F32(exp_time_min, config->args, "-exp_time_min", false, false); + PXOPT_LOOKUP_F32(exp_time_max, config->args, "-exp_time_max", false, false); + PXOPT_LOOKUP_F32(ccd_temp_min, config->args, "-ccd_temp_min", false, false); + PXOPT_LOOKUP_F32(ccd_temp_max, config->args, "-ccd_temp_max", false, false); + PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); + PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + PXOPT_LOOKUP_BOOL(randomSubset, config->args, "-random_subset", false); + PXOPT_LOOKUP_S32(randomLimit, config->args, "-random_limit", false, false); + + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); + PXOPT_LOOKUP_TIME(time_end, config->args, "-time_end", false, false); + PXOPT_LOOKUP_TIME(use_begin, config->args, "-use_begin", false, false); + PXOPT_LOOKUP_TIME(use_end, config->args, "-use_end", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-select_exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-select_inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-select_telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-select_filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-select_uri", "uri", "=="); + PXOPT_COPY_TIME(config->args, where, "-select_dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-select_dateobs_end", "dateobs", "<="); + PXOPT_COPY_F32(config->args, where, "-select_airmass_min", "airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-select_airmass_max", "airmass", "<="); + PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_max", "sat_pixel_frac", "<="); + PXOPT_COPY_F32(config->args, where, "-select_exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-select_exp_time_max", "exp_time", "<="); + PXOPT_COPY_F64(config->args, where, "-select_ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-select_ccd_temp_max", "ccd_temp", "<="); + PXOPT_COPY_F64(config->args, where, "-select_pon_time_min", "pon_time", ">="); + PXOPT_COPY_F64(config->args, where, "-select_pon_time_max", "pon_time", "<="); + PXOPT_COPY_F64(config->args, where, "-select_posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-select_posang_max", "posang", "<="); + PXOPT_COPY_F64(config->args, where, "-select_solang_min", "solang", ">="); + PXOPT_COPY_F64(config->args, where, "-select_solang_max", "solang", "<="); + + if (!psListLength(where->list)) { + psFree(where); + where = NULL; + } + + // there is some namespace overlap between the names of the fields we'd + // like to search by to setup a detrun and the names of the fields we'd + // like to assign values to so I've separated them but prepending set- to + // the assigned values + + // search for rawExps with the specified options + psArray *detrendExps = rawExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + // make sure that we found at least one rawExp + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(detrendExps)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(detrendExps); + return true; + } + + if (randomSubset && (randomLimit < detrendExps->n)) { + // generate a random-valued vector, return an index sorted by the random values + psVector *randomVector = psVectorAlloc(detrendExps->n, PS_TYPE_F32); // random values + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + for (int i = 0; i < randomVector->n; i++) { + randomVector->data.F32[i] = psRandomUniform(rng); + } + psVector *indexVector = psVectorSortIndex(NULL, randomVector); + + // accept for first n of the sequence + psArray *subset = psArrayAlloc (randomLimit); + for (int i = 0; i < randomLimit; i++ ){ + int j = indexVector->data.S32[i]; + subset->data[i] = psMemIncrRefCounter (detrendExps->data[j]); + } + psFree (detrendExps); + detrendExps = subset; + } + + // check to see if -filelevel was set on the command line + if (!filelevel) { + filelevel = psStringCopy(((rawExpRow *)detrendExps->data[0])->filelevel); + } + + if (pretend) { + // negative simple so the default is true + if (!rawExpPrintObjects(stdout, detrendExps, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detrendExps); + return false; + } + psFree(detrendExps); + return true; + } + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detrendExps); + return false; + } + + // the first iteration is always 0 + // XXX det_id + detRunInsert(config->dbh, + 0, // det_id + 0, // iteration + det_type, + mode, + "run", // state + filelevel, + workdir, + camera, + telescope, + "NA", + reduction, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + ref_det_id, + ref_iter + ); + psS64 det_id = psDBLastInsertID(config->dbh); + + // create new detInputExp row(s) from the rawExp row(s) + psArray *inputExps = psArrayAllocEmpty(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + detInputExpRow *inputExp = rawDetrenTodetInputExpRow( + detrendExps->data[i], + det_id, + 0 // the first iteration is explicitly 0 + ); + psArrayAdd(inputExps, 0, inputExp); + psFree(inputExp); + } + + psFree(detrendExps); + + // insert detInputExp objects into the database + if (!detInputExpInsertObjects(config->dbh, inputExps)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(inputExps); + return false; + } + psFree(inputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + // print the new det_id + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", det_id); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +static bool definebydetrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX this mode is not well-tested: probably need to specify iteration here + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_STR(det_type, config->args, "-set_det_type", false, false); // optional + + PXOPT_LOOKUP_STR(mode, config->args, "-set_mode", false, false); // optional + if (!isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalid mode"); + return false; + } + + // get -ref_det_id and -ref_iter : required for 'verify' mode / disallowed otherwise + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", false, false); + PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", false, false); + if (!strcmp(mode, "verify") && ((ref_det_id == 0) || (ref_iter == -1))) { + psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); + return false; + } + if (strcmp(mode, "verify") && ((ref_det_id != 0) || (ref_iter != -1))) { + psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); + return false; + } + + // the new detRun may have different values for these limits: + PXOPT_LOOKUP_STR(exp_type, config->args, "-set_exp_type", false, false); + PXOPT_LOOKUP_STR(filelevel, config->args, "-set_filelevel", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-set_filter", false, false); + PXOPT_LOOKUP_F32(airmass_min, config->args, "-set_airmass_min", false, false); + PXOPT_LOOKUP_F32(airmass_max, config->args, "-set_airmass_max", false, false); + PXOPT_LOOKUP_F32(exp_time_min, config->args, "-set_exp_time_min", false, false); + PXOPT_LOOKUP_F32(exp_time_max, config->args, "-set_exp_time_max", false, false); + PXOPT_LOOKUP_F64(ccd_temp_min, config->args, "-set_ccd_temp_min", false, false); + PXOPT_LOOKUP_F64(ccd_temp_max, config->args, "-set_ccd_temp_max", false, false); + PXOPT_LOOKUP_F64(posang_min, config->args, "-set_posang_min", false, false); + PXOPT_LOOKUP_F64(posang_max, config->args, "-set_posang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-set_solang_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-set_solang_max", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_TIME(time_begin, config->args, "-set_time_begin", false, false); + PXOPT_LOOKUP_TIME(time_end, config->args, "-set_time_end", false, false); + PXOPT_LOOKUP_TIME(use_begin, config->args, "-set_use_begin", false, false); + PXOPT_LOOKUP_TIME(use_end, config->args, "-set_use_end", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // optional + + // reference iteration to use (otherwise last iteration) + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); // for specifying input exp restrictions + + // lookup the detRun that we will be basing this one on + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = detRunSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "database error: no matching det_id found"); + return false; + } + + // sanity check the result... we should have only found one det_id + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", det_id); + } + + // pull the detRun object out the result array + detRunRow *detRun = psMemIncrRefCounter(detRuns->data[0]); + + // discard the resultarray + psFree(detRuns); + + if (iteration < 0) { + iteration = detRun->iteration; + } + + detRun->det_id = 0; // set the det_id to 0/NULL so the database can assign it + detRun->iteration = 0; // reset the iteration to 0 + + detRun->ref_det_id = ref_det_id; // not inherited: may only be set for 'verify' run + detRun->ref_iter = ref_iter; // not inherited: may only be set for 'verify' run + + // reset the state to "run" + psFree(detRun->state); + detRun->state = psStringCopy("run"); + + // walk through the optional values and update the detRun as required. Otherwise, these + // value are inherited from the specified detRun + if (det_type) { + psFree(detRun->det_type); + detRun->det_type = psStringCopy(det_type); + } + + if (mode) { + psFree(detRun->mode); + detRun->mode = psStringCopy(mode); + } + + if (exp_type) { + psFree(detRun->exp_type); + detRun->exp_type = psStringCopy(exp_type); + } + + if (filelevel) { + psFree(detRun->filelevel); + detRun->filelevel = psStringCopy(filelevel); + } + + if (workdir) { + psFree(detRun->workdir); + detRun->workdir = psStringCopy(workdir); + } + + if (filter) { + psFree(detRun->filter); + detRun->filter = psStringCopy(filter); + } + + if (!isnan(airmass_min)) { + detRun->airmass_min = airmass_min; + } + + if (!isnan(airmass_max)) { + detRun->airmass_max = airmass_max; + } + + if (!isnan(exp_time_min)) { + detRun->exp_time_min = exp_time_min; + } + + if (!isnan(exp_time_max)) { + detRun->exp_time_max = exp_time_max; + } + + if (!isnan(ccd_temp_min)) { + detRun->ccd_temp_min = ccd_temp_min; + } + + if (!isnan(ccd_temp_max)) { + detRun->ccd_temp_max = ccd_temp_max; + } + + if (!isnan(posang_min)) { + detRun->posang_min = posang_min; + } + + if (!isnan(posang_max)) { + detRun->posang_max = posang_max; + } + + if (!isnan(solang_min)) { + detRun->solang_min = solang_min; + } + + if (!isnan(solang_max)) { + detRun->solang_max = solang_max; + } + + if (label) { + psFree(detRun->label); + detRun->label = label; + } + + if (registered) { + psFree(detRun->registered); + detRun->registered = psMemIncrRefCounter(registered); + } + + if (time_begin) { + psFree(detRun->time_begin); + detRun->time_begin = psMemIncrRefCounter(time_begin); + } + + if (time_end) { + psFree(detRun->time_end); + detRun->time_end = psMemIncrRefCounter(time_end); + } + + if (use_begin) { + psFree(detRun->use_begin); + detRun->use_begin = psMemIncrRefCounter(use_begin); + } + + if (use_end) { + psFree(detRun->use_end); + detRun->use_end = psMemIncrRefCounter(use_end); + } + + if (reduction) { + psFree(detRun->reduction); + detRun->reduction = psStringCopy(reduction); + } + + // create a metadata to restrict detInputExp's be in in the specified range + + psMetadata *input_filter = psMetadataAlloc(); + + // additional restriction on the detInputExp's to be selected + PXOPT_LOOKUP_TIME(input_begin, config->args, "-set_input_begin", false, false); + if (input_begin) { + PXOPT_COPY_TIME(config->args, input_filter, "-set_input_begin", "dateobs", ">="); + } + PXOPT_LOOKUP_TIME(input_end, config->args, "-set_input_end", false, false); + if (input_end) { + PXOPT_COPY_TIME(config->args, input_filter, "-set_input_end", "dateobs", "<"); + } + PXOPT_LOOKUP_BOOL(only_accepted, config->args, "-only_accepted", false); // optional + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(input_filter); + psFree(detRun); + return false; + } + + if (!detRunInsertObject(config->dbh, detRun)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(input_filter); + psFree(detRun); + return false; + } + psFree(detRun); + + // get the det_id + psS64 newDet_id = psDBLastInsertID(config->dbh); + + psString query = pxDataGet("dettool_definebydetrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (input_filter->list->n) { + psString whereClause = psDBGenerateWhereConditionSQL(input_filter, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(input_filter); + + if (only_accepted) { + psStringAppend(&query, " AND accept = 1"); + } + + if (!p_psDBRunQuery(config->dbh, query, (long long) newDet_id, (long long) det_id, iteration)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // print the new det_id + psArray *newDetRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", newDet_id); + newDetRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!newDetRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check the result... we should have only found one det_id + if (psArrayLength(newDetRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", newDet_id); + return false; // unreachable + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, newDetRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(newDetRuns); + return false; + } + psFree(newDetRuns); + + return true; +} + +static bool runsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-mode", "mode", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + // airmass_min < airmass < airmass_max + PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_min", "<="); + PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_max", ">="); + + // exp_time_min < exp_time < exp_time_max + PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_min", "<="); + PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_max", ">="); + + // ccd_temp_min < ccd_temp < ccd_temp_max + PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_min", "<="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_max", ">="); + + PXOPT_COPY_F64(config->args, where, "-posang", "posang_min", "<="); + PXOPT_COPY_F64(config->args, where, "-posang", "posang_max", ">="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(active, config->args, "-active", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("dettool_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list) && active) { + psString whereClause = psDBGenerateWhereSQL(where, "detRun"); + psStringAppend(&query, " %s AND (detRun.state = 'run' OR detRun.state = 'stop' OR detRun.state = 'register')", whereClause); + psFree(whereClause); + } + if (psListLength(where->list) && !active) { + psString whereClause = psDBGenerateWhereSQL(where, "detRun"); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + if (!psListLength(where->list) && active) { + psStringAppend(&query, " WHERE (detRun.state = 'run' OR detRun.state = 'stop' OR detRun.state = 'register')"); + } + psFree (where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *runs = p_psDBFetchResult(config->dbh); + if (!runs) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(runs)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(runs); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, runs, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(runs); + return false; + } + + psFree(runs); + + return true; +} + +static bool childlessrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("dettool_childlessrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree (where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS64 det_id, psS32 iteration) +{ + PS_ASSERT_PTR_NON_NULL(rawExp, NULL); + + return detInputExpRowAlloc( + det_id, + iteration, + rawExp->exp_id, + true // use + ); +} + +static bool inputMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_input.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, "detInputExp"); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree (where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detInputExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool rawMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + + psString query = pxDataGet("dettool_raw.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree (where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updatedetrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_BOOL(again, config->args, "-again", false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + // either -rerun or -state must be specified + if (!(again || state)) { + psError(PS_ERR_UNKNOWN, true, "either -again or -state must be specified"); + return false; + } + if (again && state) { + psError(PS_ERR_UNKNOWN, true, "-accept and -reject are exclusive"); + return false; + } + + if (state) { + // set detRun.state to state + return setDetRunState(config, det_id, state); + } + + // else + // -again + if (!startNewIteration(config, det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to start new iteration"); + return false; + } + + return true; +} + +// used by updatedetrunMode +bool startNewIteration(pxConfig *config, psS64 det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_start_new_iteration.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // restrict the update only to the specified det_id + psStringAppend(&query, " WHERE det_id = %" PRId64 , det_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "det_id %" PRId64 " not found", det_id); + psFree(output); + return false; + } + + // start a transaction so we don't end up with an incremented iteration + // count but no detInputExps + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // up the detRuns iteration count for for the single det_id we are + // operating on + // XXX this will have to changed in order to support multiple det_ids with + // a single invocation of this functions + psS32 newIteration = incrementIteration(config, det_id); + if (!newIteration) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + bool status = false; + psS64 exp_id = psMetadataLookupS64(&status, row, "exp_id"); + if (!status) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psFree(output); + return false; + } + bool accept = psMetadataLookupBool(&status, row, "accept"); + if (!status) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for accept"); + psFree(output); + return false; + } + + // detResidExp.include is used to set detInputExp.include + if (!detInputExpInsert( + config->dbh, + det_id, + newIteration, + exp_id, + accept + ) + ) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + } + + psFree(output); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool rerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + + // we have to support multipe exp_ids + psMetadataItem *item = psMetadataLookup(config->args, "-exp_id"); + if (!item) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + + psList *exp_id_list = item->data.list; + psMetadata *where = psMetadataAlloc(); + // make sure that -exp_id was parsed correctly + // XXX this can be removed someday + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psString exp_id = mItem->data.V; + // if exp_id is NULL then it means that -exp_id has not been + // specified + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, + "at least one -exp_id is required"); + psFree(where); + return false; + } + + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } else { + psAbort("-exp_id was not parsed correctly (this should not happen"); + } + + // check that the specified exp_ids actually exist in the iteration zero + // detInputExp set + + // add the det_id & iteration == 0 to the where clause + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return false; + } + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", 0)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + return false; + } + + psArray *detrendExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + psFree(where); + return false; + } + + // build a hash for the valid exp_ids + psHash *valid_exp_ids = psHashAlloc(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + psString exp_idStr = psDBIntToString(((detInputExpRow *)detrendExps->data[i])->exp_id); + psHashAdd(valid_exp_ids, exp_idStr, detrendExps->data[i]); + psFree(exp_idStr); + } + psFree(detrendExps); + + // start a transaction so we don't end up with an incremented iteration + // count but no detInputExps + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(valid_exp_ids); + return false; + } + + // up the detRuns iteration count + psS32 newIteration = incrementIteration(config, det_id); + if (!newIteration) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(valid_exp_ids); + return false; + } + + // check exp_ids and build up an array of new detInputExp rows at the same + // time + psListIterator *iter = psListIteratorAlloc(exp_id_list, 0, false); + psMetadataItem *mItem = NULL; + psArray *newInputExps = psArrayAllocEmpty(psListLength(exp_id_list)); + while ((mItem = psListGetAndIncrement(iter))) { + detInputExpRow *inputExp = psHashLookup(valid_exp_ids, + (char *)mItem->data.V); + if (!inputExp) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + // invalid exp_id + psError(PS_ERR_UNKNOWN, false, "exp_id %s is invalid for det_id %" PRId64, + (char *)mItem->data.V, det_id); + psFree(iter); + psFree(valid_exp_ids); + return false; + } + detInputExpRow *newInputExp = detInputExpRowAlloc( + det_id, + newIteration, + inputExp->exp_id, + true // use + ); + psArrayAdd(newInputExps, 0, newInputExp); + psFree(newInputExp); + } + psFree(iter); + psFree(valid_exp_ids); + + for (long i = 0; i < psArrayLength(newInputExps); i++) { + if (!detInputExpInsertObject(config->dbh, newInputExps->data[i])) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(newInputExps); + return false; + } + } + psFree(newInputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool register_detrendMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); // required + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", true, false); // required + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_F32(airmass_min, config->args, "-airmass_min", false, false); + PXOPT_LOOKUP_F32(airmass_max, config->args, "-airmass_max", false, false); + PXOPT_LOOKUP_F32(exp_time_min, config->args, "-exp_time_min", false, false); + PXOPT_LOOKUP_F32(exp_time_max, config->args, "-exp_time_max", false, false); + PXOPT_LOOKUP_F32(ccd_temp_min, config->args, "-ccd_temp_min", false, false); + PXOPT_LOOKUP_F32(ccd_temp_max, config->args, "-ccd_temp_max", false, false); + PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); + PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); + PXOPT_LOOKUP_TIME(time_end, config->args, "-time_end", false, false); + PXOPT_LOOKUP_TIME(use_begin, config->args, "-use_begin", false, false); + PXOPT_LOOKUP_TIME(use_end, config->args, "-use_end", false, false); + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", false, false); + PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!detRunInsert(config->dbh, + 0, // det_id + 0, // the iteration is fixed at 0 + det_type, + "register", // mode is required to be "register" + "register", // state + filelevel, + workdir, + camera, + telescope, + exp_type, + NULL, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + ref_det_id, + ref_iter + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // print the new detRun + psS64 det_id = psDBLastInsertID(config->dbh); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 "(this should not happen)", det_id); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +// NOTE : this function is also used by addcorrectimfileMode +bool register_detrend_imfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // required + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); // required + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + if (!detRegisteredImfileInsert(config->dbh, + det_id, + 0, // the iteration is fixed at 0 + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + 0 // fault code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psS32 incrementIteration(pxConfig *config, psS64 det_id) +{ + // this function returns zero on error + PS_ASSERT_PTR_NON_NULL(config, 0); + + char *query = "UPDATE detRun SET iteration = iteration + 1 WHERE det_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to increment iteration for det_id %" PRId64, det_id); + return 0; + } + + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return 0; + } + + psArray *detRuns = detRunSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "no detRun rows found"); + return 0; + } + + // sanity check the database + if (psArrayLength(detRuns) != 1) { + // this should no happen + psAbort( "database query return too many rows (this should not happen"); + } + + psS32 newIteration = ((detRunRow *)detRuns->data[0])->iteration; + psFree(detRuns); + + return newIteration; +} + +bool setDetRunState(pxConfig *config, psS64 det_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + if (!isValidDetRunState (state)) return false; + + char *query = "UPDATE detRun SET state = '%s' WHERE det_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for det_id %" PRId64, det_id); + return false; + } + + return true; +} + +// the detRun states are a superset of the data states below +bool isValidDetRunState (const char *state) { + + // check that state is a valid string value + if (!strcmp(state, "run")) return true; + if (!strcmp(state, "stop")) return true; + if (!strcmp(state, "drop")) return true; + if (!strcmp(state, "wait")) return true; + if (!strcmp(state, "test")) return true; + if (!strcmp(state, "ignore")) return true; + if (!strcmp(state, "register")) return true; + + psError(PS_ERR_UNKNOWN, true, "invalid detRun state: %s", state); + return false; +} + +bool isValidDataState (const char *data_state) { + + // check that state is a valid string value + if (!strcmp(data_state, "run")) return true; + if (!strcmp(data_state, "stop")) return true; + if (!strcmp(data_state, "drop")) return true; + if (!strcmp(data_state, "register")) return true; + + psError(PS_ERR_UNKNOWN, true, "invalid data state: %s", data_state); + return false; +} + +bool isValidMode(pxConfig *config, const char *mode) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(mode, false); + + // check that state is a valid string value + if (!strcmp(mode, "master")) return true; + if (!strcmp(mode, "verify")) return true; + + psError(PS_ERR_UNKNOWN, false, "invalid detRun mode: %s", mode); + return false; +} + + +bool setProcessedImfileDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *class_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + PS_ASSERT_PTR_NON_NULL(class_id, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detProcessedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND exp_id = %" PRId64 + " AND class_id = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, exp_id, class_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", exp_id %" PRId64 ", class_id %s", + det_id, exp_id, class_id); + return false; + } + + return true; +} + +bool setProcessedExpDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detProcessedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND exp_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", exp_id %" PRId64, + det_id, exp_id); + return false; + } + + return true; +} + + +bool setStackedImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(class_id, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detStackedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = %s"; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration, class_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 "class_id %s", + det_id, iteration, class_id); + return false; + } + + return true; +} + +bool setNormStatImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(class_id, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detNormalizedStatImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = %s"; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, + det_id, iteration); + return false; + } + + return true; +} + +bool setNormImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(class_id, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detNormalizedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = %s"; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration, class_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 " class %s", + det_id, iteration, class_id); + return false; + } + + return true; +} + +bool setNormExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detNormalizedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, + det_id, iteration); + return false; + } + + return true; +} + +bool setResidImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *class_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + PS_ASSERT_PTR_NON_NULL(class_id, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detResidImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND exp_id = %" PRId64 + " AND class_id = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration, exp_id, class_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 ", exp_id %" PRId64 ", class_id %s", + det_id, iteration, exp_id, class_id); + return false; + } + + return true; +} + +bool setResidExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detResidExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND exp_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 ", exp_id %" PRId64, + det_id, iteration, exp_id); + return false; + } + + return true; +} + +#if 0 +// XXX this function was left in commented as this method may be useful in the +// future +static psArray *validDetInputClassIds(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *rawImfiles = searchInputImfiles(config, det_id); + if (!rawImfiles) { + return NULL; + } + + psArray *valid_class_ids = NULL; + { + // All this jumping through hoops is so that we end up with unique + // values. PP thinks that making multiple passes through this array and + // deleting matched elements would end up being more expensive then a + // double sort and stagger scheme. JH thinks it would be cheaper to + // just do a unqiue sort and delete. So this is really just a cheap + // hack to avoid implimenting a unique sort function but at least it's + // stable. + psHash *hash = psHashAlloc(psArrayLength(rawImfiles)); + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + psHashAdd(hash, + ((rawImfileRow *)rawImfiles->data[i])->class_id, + ((rawImfileRow *)rawImfiles->data[i])->class_id + ); + } + valid_class_ids = psHashToArray(hash); + psFree(hash); + } + psFree(rawImfiles); + + return valid_class_ids; +} + +static psArray *searchInputImfiles(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *inputExps = NULL; + { + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS32)atoi(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return NULL; + } + inputExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!inputExps) { + psError(PS_ERR_UNKNOWN, false, "no detInputExp rows found"); + return NULL; + } + + // find rawImfiles associated with detInputExps + psArray *rawImfiles = NULL; + { + psMetadata *where = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(inputExps); i++) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", + ((detInputExpRow *)inputExps->data[i])->exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(inputExps); + psFree(where); + return NULL; + } + } + psFree(inputExps); + rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + // XXX this really should be sorted for uniqueness + psFree(where); + } + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} +#endif + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool.h (revision 22235) @@ -0,0 +1,226 @@ +/* + * dettool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#ifndef DETTOOL_H +#define DETTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + DETTOOL_MODE_NONE = PXTOOL_MODE_NONE, + DETTOOL_MODE_PENDING, + DETTOOL_MODE_DEFINEBYTAG, + DETTOOL_MODE_DEFINEBYQUERY, + DETTOOL_MODE_DEFINEBYDETRUN, + DETTOOL_MODE_MAKECORRECTION, + DETTOOL_MODE_TOCORRECTEXP, + DETTOOL_MODE_TOCORRECTIMFILE, + DETTOOL_MODE_ADDCORRECTIMFILE, + DETTOOL_MODE_RUNS, + DETTOOL_MODE_CHILDLESSRUN, + DETTOOL_MODE_INPUT, + DETTOOL_MODE_RAW, + + DETTOOL_MODE_TOPROCESSEDIMFILE, + DETTOOL_MODE_ADDPROCESSEDIMFILE, + DETTOOL_MODE_PROCESSEDIMFILE, + DETTOOL_MODE_REVERTPROCESSEDIMFILE, + DETTOOL_MODE_UPDATEPROCESSEDIMFILE, + DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDIMFILE, + DETTOOL_MODE_DONECLEANUP_PROCESSEDIMFILE, + + DETTOOL_MODE_TOPROCESSEDEXP, + DETTOOL_MODE_ADDPROCESSEDEXP, + DETTOOL_MODE_PROCESSEDEXP, + DETTOOL_MODE_REVERTPROCESSEDEXP, + DETTOOL_MODE_UPDATEPROCESSEDEXP, + DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, + DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, + + DETTOOL_MODE_TOSTACKED, + DETTOOL_MODE_ADDSTACKED, + DETTOOL_MODE_STACKED, + DETTOOL_MODE_REVERTSTACKED, + DETTOOL_MODE_UPDATESTACKED, + DETTOOL_MODE_PENDINGCLEANUP_STACKED, + DETTOOL_MODE_DONECLEANUP_STACKED, + + DETTOOL_MODE_TONORMALIZEDSTAT, + DETTOOL_MODE_ADDNORMALIZEDSTAT, + DETTOOL_MODE_NORMALIZEDSTAT, + DETTOOL_MODE_REVERTNORMALIZEDSTAT, + DETTOOL_MODE_UPDATENORMALIZEDSTAT, + DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDSTAT, + DETTOOL_MODE_DONECLEANUP_NORMALIZEDSTAT, + + DETTOOL_MODE_TONORMALIZE, + DETTOOL_MODE_ADDNORMALIZEDIMFILE, + DETTOOL_MODE_NORMALIZEDIMFILE, + DETTOOL_MODE_REVERTNORMALIZEDIMFILE, + DETTOOL_MODE_UPDATENORMALIZEDIMFILE, + DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDIMFILE, + DETTOOL_MODE_DONECLEANUP_NORMALIZEDIMFILE, + + DETTOOL_MODE_TONORMALIZEDEXP, + DETTOOL_MODE_ADDNORMALIZEDEXP, + DETTOOL_MODE_NORMALIZEDEXP, + DETTOOL_MODE_REVERTNORMALIZEDEXP, + DETTOOL_MODE_UPDATENORMALIZEDEXP, + DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDEXP, + DETTOOL_MODE_DONECLEANUP_NORMALIZEDEXP, + + DETTOOL_MODE_TORESIDIMFILE, + DETTOOL_MODE_ADDRESIDIMFILE, + DETTOOL_MODE_RESIDIMFILE, + DETTOOL_MODE_REVERTRESIDIMFILE, + DETTOOL_MODE_UPDATERESIDIMFILE, + DETTOOL_MODE_PENDINGCLEANUP_RESIDIMFILE, + DETTOOL_MODE_DONECLEANUP_RESIDIMFILE, + + DETTOOL_MODE_TORESIDEXP, + DETTOOL_MODE_ADDRESIDEXP, + DETTOOL_MODE_RESIDEXP, + DETTOOL_MODE_REVERTRESIDEXP, + DETTOOL_MODE_UPDATERESIDEXP, + DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, + DETTOOL_MODE_DONECLEANUP_RESIDEXP, + + DETTOOL_MODE_TODETRUNSUMMARY, + DETTOOL_MODE_ADDDETRUNSUMMARY, + DETTOOL_MODE_DETRUNSUMMARY, + DETTOOL_MODE_REVERTDETRUNSUMMARY, + DETTOOL_MODE_UPDATEDETRUNSUMMARY, + DETTOOL_MODE_UPDATEDETRUN, + DETTOOL_MODE_RERUN, + DETTOOL_MODE_REGISTER_DETREND, + DETTOOL_MODE_REGISTER_DETREND_IMFILE +} dettoolMode; + +pxConfig *dettoolConfig(pxConfig *config, int argc, char **argv); + +// correct +bool makecorrectionMode(pxConfig *config); +bool tocorrectexpMode(pxConfig *config); +bool tocorrectimfileMode(pxConfig *config); +bool addcorrectimfileMode(pxConfig *config); + +// register +bool register_detrend_imfileMode(pxConfig *config); + +// processedimfile +bool toprocessedimfileMode(pxConfig *config); +bool addprocessedimfileMode(pxConfig *config); +bool processedimfileMode(pxConfig *config); +bool revertprocessedimfileMode(pxConfig *config); +bool updateprocessedimfileMode(pxConfig *config); +bool pendingcleanup_processedimfileMode(pxConfig *config); +bool donecleanup_processedimfileMode(pxConfig *config); + +// processedexp +bool toprocessedexpMode(pxConfig *config); +bool addprocessedexpMode(pxConfig *config); +bool processedexpMode(pxConfig *config); +bool revertprocessedexpMode(pxConfig *config); +bool updateprocessedexpMode(pxConfig *config); +bool pendingcleanup_processedexpMode(pxConfig *config); +bool donecleanup_processedexpMode(pxConfig *config); + +// stackedimfile +bool tostackedMode(pxConfig *config); +bool addstackedMode(pxConfig *config); +bool stackedMode(pxConfig *config); +bool revertstackedMode(pxConfig *config); +bool updatestackedMode(pxConfig *config); +bool pendingcleanup_stackedMode(pxConfig *config); +bool donecleanup_stackedMode(pxConfig *config); + +// normalizedstat +bool tonormalizedstatMode(pxConfig *config); +bool addnormalizedstatMode(pxConfig *config); +bool normalizedstatMode(pxConfig *config); +bool revertnormalizedstatMode(pxConfig *config); +bool updatenormalizedstatMode(pxConfig *config); +bool pendingcleanup_normalizedstatMode(pxConfig *config); +bool donecleanup_normalizedstatMode(pxConfig *config); + +// normalizedimfile +bool tonormalizeMode(pxConfig *config); +bool addnormalizedimfileMode(pxConfig *config); +bool normalizedimfileMode(pxConfig *config); +bool revertnormalizedimfileMode(pxConfig *config); +bool updatenormalizedimfileMode(pxConfig *config); +bool pendingcleanup_normalizedimfileMode(pxConfig *config); +bool donecleanup_normalizedimfileMode(pxConfig *config); + +// normalizedexp +bool tonormalizedexpMode(pxConfig *config); +bool addnormalizedexpMode(pxConfig *config); +bool normalizedexpMode(pxConfig *config); +bool revertnormalizedexpMode(pxConfig *config); +bool updatenormalizedexpMode(pxConfig *config); +bool pendingcleanup_normalizedexpMode(pxConfig *config); +bool donecleanup_normalizedexpMode(pxConfig *config); + +// residimfile +bool toresidimfileMode(pxConfig *config); +bool addresidimfileMode(pxConfig *config); +bool residimfileMode(pxConfig *config); +bool revertresidimfileMode(pxConfig *config); +bool updateresidimfileMode(pxConfig *config); +bool pendingcleanup_residimfileMode(pxConfig *config); +bool donecleanup_residimfileMode(pxConfig *config); + +// residexp +bool toresidexpMode(pxConfig *config); +bool addresidexpMode(pxConfig *config); +bool residexpMode(pxConfig *config); +bool revertresidexpMode(pxConfig *config); +bool updateresidexpMode(pxConfig *config); +bool pendingcleanup_residexpMode(pxConfig *config); +bool donecleanup_residexpMode(pxConfig *config); + +// detrunsummary +bool todetrunsummaryMode(pxConfig *config); +bool adddetrunsummaryMode(pxConfig *config); +bool detrunsummaryMode(pxConfig *config); +bool revertdetrunsummaryMode(pxConfig *config); +bool updatedetrunsummaryMode(pxConfig *config); + +// other utilities +bool startNewIteration(pxConfig *config, psS64 det_id); +bool setDetRunState(pxConfig *config, psS64 det_id, const char *state); +bool isValidDetRunState (const char *state); +bool isValidDataState (const char *data_state); +bool isValidMode(pxConfig *config, const char *mode); + +// functions to set the 'data_state' for stages +bool setProcessedImfileDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *class_id, const char *data_state); +bool setProcessedExpDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *data_state); +bool setStackedImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); +bool setNormStatImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); +bool setNormImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); +bool setNormExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *data_state); +bool setResidImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *class_id, const char *data_state); +bool setResidExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *data_state); + +#endif // DETTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettoolConfig.c (revision 22235) @@ -0,0 +1,972 @@ +/* + * dettoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +pxConfig *dettoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -pending + // XXX EAM : is this used? does it make sense? + psMetadata *pendingArgs = psMetadataAlloc(); + psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri", 0, "search by URL", NULL); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definebytag + psMetadata *definebytagArgs = psMetadataAlloc(); + psMetadataAddS64(definebytagArgs, PS_LIST_TAIL, "-exp_id", PS_META_DUPLICATE_OK, "include this exposure (multiple OK, required)", 0); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-det_type", 0, "define the type of detrend run (required)", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-mode", 0, "define the mode of this detrend run (master, verify)", "master"); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-filelevel", 0, "define filelevel", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-inst", 0, "define camera", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-filter", 0, "define filter ", NULL); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-time_end", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-use_begin", 0, "start of detrend run applicable period", NULL); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-use_end", 0, "end of detrend run applicable period", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class for processing", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-label", 0, "define detrun label", NULL); + psMetadataAddBool(definebytagArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadataAddS64(definebytagArgs, PS_LIST_TAIL, "-ref_det_id", 0, "define reference detrend (verify mode only)", 0); + psMetadataAddS32(definebytagArgs, PS_LIST_TAIL, "-ref_iter", 0, "define reference detrend (verify mode only)", -1); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-det_type", 0, "define the type of detrend run (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-mode", 0, "define the mode of this detrend run (master, verify)", "master"); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, "define filelevel", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-inst", 0, "define camera (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "define filter ", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-time_end", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-use_begin", 0, "start of detrend run applicable period", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-use_end", 0, "end of detrend run applicable period", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, "search for exp_type", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, "search for camera", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_telescope", 0, "search for telescope", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", 0, "search for filter", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_uri", 0, "search for uri", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max", 0, "define max airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_pon_time_min", 0, "define min power-on time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_pon_time_max", 0, "define max power-on time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, "define max solar angle", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class for processing", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define detrun label", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-random_subset", 0, "use a random subset of elements", false); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random_limit", 0, "use this number of random elements", 20); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-ref_det_id", 0, "define reference detrend (verify mode only)", 0); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-ref_iter", 0, "define reference detrend (verify mode only)", -1); + + // -definebydetrun + psMetadata *definebydetrunArgs = psMetadataAlloc(); + psMetadataAddS64(definebydetrunArgs, PS_LIST_TAIL, "-det_id", 0, "det ID to base a new detRun on (required)", 0); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_det_type", 0, "define the type of detrend run", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_mode", 0, "define the mode of this detrend run", "master"); + psMetadataAddS64(definebydetrunArgs, PS_LIST_TAIL, "-ref_det_id", 0, "reference det ID ('verify' run only)", 0); + psMetadataAddS32(definebydetrunArgs, PS_LIST_TAIL, "-ref_iter", 0, "reference iteration ('verify' run only)", -1); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_type", 0, "define exposure type", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_filelevel", 0, "define filelevel", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_filter", 0, "define filter ", NULL); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_airmass_min", 0, "define airmass", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_airmass_max", 0, "define airmass", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_time_min", 0, "define exposure time", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_time_max", 0, "define exposure time", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_ccd_temp_min", 0, "define ccd tempature", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_ccd_temp_max", 0, "define ccd tempature", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_min", 0, "define rotator position angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_max", 0, "define rotator position angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_min", 0, "define solar angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_max", 0, "define solar angle", NAN); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_time_begin", 0, "start of period to apply detrend too", NULL); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_time_end", 0, "end of period to apply detrend too", NULL); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_use_begin", 0, "start of detrend run applicable period", NULL); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_use_end", 0, "end of detrend run applicable period", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class for processing", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_label", 0, "define detrun label", NULL); + psMetadataAddBool(definebydetrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_input_begin", 0, "input exp must be in this period", NULL); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_input_end", 0, "input exp must be in this period", NULL); + psMetadataAddBool(definebydetrunArgs, PS_LIST_TAIL, "-only_accepted", 0, "limit input exp to those accepted in the source detRun", false); + psMetadataAddS32(definebydetrunArgs, PS_LIST_TAIL, "-iteration", 0, "det ID to base a new detRun on", -1); + + // *** these are in dettool_correction.c *** + // -makecorrection + psMetadata *makecorrectionArgs = psMetadataAlloc(); + psMetadataAddStr(makecorrectionArgs, PS_LIST_TAIL, "-det_type", 0, "set detrend type of corrected detRun", NULL); + psMetadataAddS64(makecorrectionArgs, PS_LIST_TAIL, "-det_id", 0, "detRun det_id to be corrected (required)", 0); + psMetadataAddS32(makecorrectionArgs, PS_LIST_TAIL, "-iteration", 0, "detRun iteration to be corrected (required)", -1);; + psMetadataAddBool(makecorrectionArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadataAddStr(makecorrectionArgs, PS_LIST_TAIL, "-workdir", 0, "set workdir", NULL); + psMetadataAddStr(makecorrectionArgs, PS_LIST_TAIL, "-reduction", 0, "set reduction", NULL); + psMetadataAddStr(makecorrectionArgs, PS_LIST_TAIL, "-label", 0, "set label", NULL); + + // -tocorrectexp + psMetadata *tocorrectexpArgs = psMetadataAlloc(); + psMetadataAddU64(tocorrectexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocorrectexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tocorrectimfile + psMetadata *tocorrectimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddStr(tocorrectimfileArgs, PS_LIST_TAIL, "-det_type", 0, "search by type of corrected detRun", NULL); + psMetadataAddU64(tocorrectimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocorrectimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addcorrectimfile + psMetadata *addcorrectimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + // -runs + psMetadata *runsArgs = psMetadataAlloc(); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-det_type", 0, "search for type of detrend run", NULL); + psMetadataAddS64(runsArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-inst", 0, "search for camera (instrument)", NULL); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(runsArgs, PS_LIST_TAIL, "-airmass", 0, "match airmass", NAN); + psMetadataAddF32(runsArgs, PS_LIST_TAIL, "-exp_time", 0, "match exp time", NAN); + psMetadataAddF32(runsArgs, PS_LIST_TAIL, "-ccd_temp", 0, "match ccd temp", NAN); + psMetadataAddF64(runsArgs, PS_LIST_TAIL, "-posang", 0, "match posang", NAN); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-mode", 0, "search for det run by mode", NULL); + psMetadataAddBool(runsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(runsArgs, PS_LIST_TAIL, "-active", 0, "only return active detRuns", false); + psMetadataAddU64(runsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -childlessrun + psMetadata *childlessrunArgs = psMetadataAlloc(); + psMetadataAddStr(childlessrunArgs, PS_LIST_TAIL, "-det_type", 0, "search for type of detrend run", NULL); + psMetadataAddU64(childlessrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(childlessrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -input + psMetadata *inputArgs = psMetadataAlloc(); + psMetadataAddS64(inputArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(inputArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number (default 0)", 0); + psMetadataAddS64(inputArgs, PS_LIST_TAIL, "-exp_id", 0, "search for exp ID", 0); + psMetadataAddBool(inputArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -raw + psMetadata *rawArgs = psMetadataAlloc(); + psMetadataAddBool(rawArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -toprocessedimfile + psMetadata *toprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS64(toprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search for exp ID", 0); + psMetadataAddStr(toprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddU64(toprocessedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toprocessedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp ID (required)", 0); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe slope (0th term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe slope (1st term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search for exp ID", 0); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-select_state", 0, "search for state", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-select_mode", 0, "search for mode", NULL); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-included", 0, "restrict results to exposures 'includeded' in the current iteration", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertprocessedimfile + psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID (required)", 0); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateprocessedimfile + psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_processedimfile + psMetadata *pendingcleanup_processedimfileArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + + // -donecleanup_processedimfile + psMetadata *donecleanup_processedimfileArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + + // -toprocessedexp + psMetadata *toprocessedexpArgs = psMetadataAlloc(); + psMetadataAddU64(toprocessedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toprocessedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addproccessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe slope (0th term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe slope (1st term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -proccessedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp ID", 0); + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertprocessedexp + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateprocessedexp + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_processedexp + psMetadata *pendingcleanup_processedexpArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + + // -donecleanup_processedexp + psMetadata *donecleanup_processedexpArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + + // -tostacked + psMetadata *tostackedArgs = psMetadataAlloc(); + psMetadataAddU64(tostackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tostackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addstacked + psMetadata *addstackedArgs = psMetadataAlloc(); + psMetadataAddS64(addstackedArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addstackedArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number (required)", 0); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddS16(addstackedArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + + // -stacked + psMetadata *stackedArgs = psMetadataAlloc(); + psMetadataAddS64(stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(stackedArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddStr(stackedArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddStr(stackedArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddU64(stackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(stackedArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(stackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertstacked + psMetadata *revertstackedArgs= psMetadataAlloc(); + psMetadataAddS64(revertstackedArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID (required)", 0); + psMetadataAddS32(revertstackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(revertstackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(revertstackedArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatestacked + psMetadata *updatestackedArgs = psMetadataAlloc(); + psMetadataAddS64(updatestackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_stacked + psMetadata *pendingcleanup_stackedArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + + // -donecleanup_stacked + psMetadata *donecleanup_stackedArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_stackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_stackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(donecleanup_stackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(donecleanup_stackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + + // -tonormalizedstat + psMetadata *tonormalizedstatArgs = psMetadataAlloc(); + psMetadataAddU64(tonormalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tonormalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addnormalizedstat + psMetadata *addnormstatArgs = psMetadataAlloc(); + psMetadataAddS64(addnormstatArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addnormstatArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddStr(addnormstatArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddF32(addnormstatArgs, PS_LIST_TAIL, "-norm", 0, "define normal value (required)", NAN); + psMetadataAddS16(addnormstatArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -normalizedstat + psMetadata *normalizedstatArgs = psMetadataAlloc(); + psMetadataAddS64(normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); + psMetadataAddS32(normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddU64(normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(normalizedstatArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertnormalizedstat + psMetadata *revertnormalizedstatArgs= psMetadataAlloc(); + psMetadataAddS64(revertnormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertnormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(revertnormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(revertnormalizedstatArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatenormalizedstat + psMetadata *updatenormalizedstatArgs = psMetadataAlloc(); + psMetadataAddS64(updatenormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_normalizedstat + psMetadata *pendingcleanup_normalizedstatArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -donecleanup_normalizedstat + psMetadata *donecleanup_normalizedstatArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -tonormalize + psMetadata *tonormalizeArgs = psMetadataAlloc(); + psMetadataAddU64(tonormalizeArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tonormalizeArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addnormalizedimfile + psMetadata *addnormalizedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddS16(addnormalizedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + // -normalizedimfile + psMetadata *normalizedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddStr(normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddStr(normalizedimfileArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddU64(normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(normalizedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertnormalizedimfile + psMetadata *revertnormalizedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(revertnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(revertnormalizedimfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatenormalizedimfile + psMetadata *updatenormalizedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updatenormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_normalizedimfile + psMetadata *pendingcleanup_normalizedimfileArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -donecleanup_normalizedimfile + psMetadata *donecleanup_normalizedimfileArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -tonormalizedexp + psMetadata *tonormalizedexpArgs = psMetadataAlloc(); + psMetadataAddU64(tonormalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tonormalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addnormalizedexp + psMetadata *addnormalizedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addnormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddS16(addnormalizedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + // -normalizedexp + psMetadata *normalizedexpArgs = psMetadataAlloc(); + psMetadataAddS64(normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); + psMetadataAddS32(normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(normalizedexpArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddU64(normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(normalizedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertnormalizedexp + psMetadata *revertnormalizedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertnormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS16(revertnormalizedexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatenormalizedexp + psMetadata *updatenormalizedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_normalizedexp + psMetadata *pendingcleanup_normalizedexpArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + + // -donecleanup_normalizedexp + psMetadata *donecleanup_normalizedexpArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + + // -toresidimfile + psMetadata *toresidimfileArgs = psMetadataAlloc(); + psMetadataAddU64(toresidimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toresidimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresidimfile + // XXX does a default value of 0 mean the entry is not instatiated? what about a supplied value of 0? + psMetadata *addresidimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddS64(addresidimfileArgs, PS_LIST_TAIL, "-ref_det_id", 0, "define reference detrend ID (required)", 0); + psMetadataAddS32(addresidimfileArgs, PS_LIST_TAIL, "-ref_iter", 0, "define reference iteration number (required)", -1); + psMetadataAddS64(addresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddS16(addresidimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_skewness", 0, "define exposure background skewness", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_kurtosis", 0, "define exposure background kurtosis", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bin_stdev", 0, "define exposure background binned stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe slope (0th term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe slope (1st term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_resid_0", 0, "define fringe residual (0th term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_resid_1", 0, "define fringe residual (1st term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_resid_2", 0, "define fringe residual (2nd term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + // -residimfile + psMetadata *residimfileArgs = psMetadataAlloc(); + psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddU64(residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-select_state", 0, "search for state", NULL); + + // -revertresidimfile + psMetadata *revertresidimfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + psMetadataAddStr(revertresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddS16(revertresidimfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateresidimfile + psMetadata *updateresidimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(updateresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_residimfile + psMetadata *pendingcleanup_residimfileArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -donecleanup_residimfile + psMetadata *donecleanup_residimfileArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(donecleanup_residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(donecleanup_residimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -toresidexp + psMetadata *toresidexpArgs = psMetadataAlloc(); + psMetadataAddU64(toresidexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toresidexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresidexp + psMetadata *addresidexpArgs = psMetadataAlloc(); + psMetadataAddS64(addresidexpArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addresidexpArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddS64(addresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, "define detrend ID (required)", 0); + psMetadataAddS16(addresidexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_skewness", 0, "define exposure background skewness", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_kurtosis", 0, "define exposure background kurtosis", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bin_stdev", 0, "define exposure background binned stdev", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe slope (0th term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe slope (1st term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_resid_0", 0, "define fringe residual (0th term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_resid_1", 0, "define fringe residual (1st term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_resid_2", 0, "define fringe residual (2nd term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddBool(addresidexpArgs, PS_LIST_TAIL, "-reject", 0, "exposure is not to be stacked in the next iteration", false); + + // -residexp + psMetadata *residexpArgs = psMetadataAlloc(); + psMetadataAddS64(residexpArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(residexpArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddS64(residexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search for exp ID", 0); + psMetadataAddStr(residexpArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-reject", 0, "search for acceptable residuals", false); + psMetadataAddU64(residexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertresidexp + psMetadata *revertresidexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertresidexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + psMetadataAddS16(revertresidexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateresidexp + psMetadata *updateresidexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateresidexpArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID", 0); + psMetadataAddS32(updateresidexpArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddS64(updateresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp ID", 0); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject", 0, "exposure is not to be stacked in the next iteration", false); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_residexp + psMetadata *pendingcleanup_residexpArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_residexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_residexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_residexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(pendingcleanup_residexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(pendingcleanup_residexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + + // -donecleanup_residexp + psMetadata *donecleanup_residexpArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_residexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_residexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_residexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(donecleanup_residexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(donecleanup_residexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + + // -todetrunsummary + psMetadata *todetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddBool(todetrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(todetrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -adddetrunsummary + psMetadata *adddetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(adddetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(adddetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddS16(adddetrunsummaryArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddBool(adddetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, "declare that this detrun iteration is accepted as a master", false); + psMetadataAddBool(adddetrunsummaryArgs, PS_LIST_TAIL, "-again", 0, "start a new iteration of this detrend run", false); + + // -detrunsummary + psMetadata *detrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(detrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(detrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddU64(detrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, "search for acceptable residuals", false); + + // -revertdetrunsummary + psMetadata *revertdetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(revertdetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertdetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS16(revertdetrunsummaryArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatedetrunsummary + psMetadata *updatedetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(updatedetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend master for detrend ID (required)", 0); + psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, "declare that this detrun iteration is accepted as a master", false); + psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, "reject this detrun iteration as a master", false); + + // -updatedetrun + psMetadata *updatedetrunArgs = psMetadataAlloc(); + psMetadataAddS64(updatedetrunArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend master for detrend ID (required)", 0); + psMetadataAddBool(updatedetrunArgs, PS_LIST_TAIL, "-again", 0, "start a new iteration of this detrend run", false); + psMetadataAddStr(updatedetrunArgs, PS_LIST_TAIL, "-state", 0, "set the state of this detrend run", false); + + // -rerun + psMetadata *rerunArgs = psMetadataAlloc(); + psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend master for detrend ID (required)", 0); + psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-exp_id", PS_META_DUPLICATE_OK, "include this exposure (multiple OK, required)", 0); + + // -register_detrend + psMetadata *register_detrendArgs = psMetadataAlloc(); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-det_type", 0, "define the type of detrend run (required)", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-filelevel", 0, "define filelevel (required)", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-inst", 0, "define camera", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-filter", 0, "define filter ", NULL); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-time_end", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-use_begin", 0, "start of detrend run applicable period", NULL); + psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-use_end", 0, "end of detrend run applicable period", NULL); + psMetadataAddS64(register_detrendArgs, PS_LIST_TAIL, "-ref_det_id", 0, "define reference det_id", 0); + psMetadataAddS32(register_detrendArgs, PS_LIST_TAIL, "-ref_iter", 0, "define reference iteration", -1); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-label", 0, "define detrun label", NULL); + psMetadataAddBool(register_detrendArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -register_detrend_imfile + psMetadata *register_detrend_imfileArgs = psMetadataAlloc(); + psMetadataAddS64(register_detrend_imfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI (required)", NULL); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-pending", "list active detruns", DETTOOL_MODE_PENDING, pendingArgs); + PXOPT_ADD_MODE("-definebytag", "", DETTOOL_MODE_DEFINEBYTAG, definebytagArgs); + PXOPT_ADD_MODE("-definebyquery", "", DETTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definebydetrun", "", DETTOOL_MODE_DEFINEBYDETRUN, definebydetrunArgs); + PXOPT_ADD_MODE("-makecorrection", "", DETTOOL_MODE_MAKECORRECTION, makecorrectionArgs); + PXOPT_ADD_MODE("-tocorrectexp", "", DETTOOL_MODE_TOCORRECTEXP, tocorrectexpArgs); + PXOPT_ADD_MODE("-tocorrectimfile", "", DETTOOL_MODE_TOCORRECTIMFILE, tocorrectimfileArgs); + PXOPT_ADD_MODE("-addcorrectimfile", "", DETTOOL_MODE_ADDCORRECTIMFILE, addcorrectimfileArgs); + PXOPT_ADD_MODE("-raw", "", DETTOOL_MODE_RAW, rawArgs); + PXOPT_ADD_MODE("-runs", "", DETTOOL_MODE_RUNS, runsArgs); + PXOPT_ADD_MODE("-childlessrun", "", DETTOOL_MODE_CHILDLESSRUN, childlessrunArgs); + PXOPT_ADD_MODE("-input", "", DETTOOL_MODE_INPUT, inputArgs); + + PXOPT_ADD_MODE("-toprocessedimfile", "", DETTOOL_MODE_TOPROCESSEDIMFILE, toprocessedimfileArgs); + PXOPT_ADD_MODE("-addprocessedimfile", "", DETTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXOPT_ADD_MODE("-processedimfile", "", DETTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXOPT_ADD_MODE("-revertprocessedimfile", "", DETTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); + PXOPT_ADD_MODE("-updateprocessedimfile", "", DETTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); + PXOPT_ADD_MODE("-pendingcleanup_processedimfile", "", DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDIMFILE, pendingcleanup_processedimfileArgs); + PXOPT_ADD_MODE("-donecleanup_processedimfile", "", DETTOOL_MODE_DONECLEANUP_PROCESSEDIMFILE, donecleanup_processedimfileArgs); + + PXOPT_ADD_MODE("-toprocessedexp", "", DETTOOL_MODE_TOPROCESSEDEXP, toprocessedexpArgs); + PXOPT_ADD_MODE("-addprocessedexp", "", DETTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "", DETTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "", DETTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "", DETTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpArgs); + PXOPT_ADD_MODE("-pendingcleanup_processedexp", "", DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, pendingcleanup_processedexpArgs); + PXOPT_ADD_MODE("-donecleanup_processedexp", "", DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, donecleanup_processedexpArgs); + + PXOPT_ADD_MODE("-tostacked", "", DETTOOL_MODE_TOSTACKED, tostackedArgs); + PXOPT_ADD_MODE("-addstacked", "", DETTOOL_MODE_ADDSTACKED, addstackedArgs); + PXOPT_ADD_MODE("-stacked", "", DETTOOL_MODE_STACKED, stackedArgs); + PXOPT_ADD_MODE("-revertstacked", "", DETTOOL_MODE_REVERTSTACKED, revertstackedArgs); + PXOPT_ADD_MODE("-updatestacked", "", DETTOOL_MODE_UPDATESTACKED, updatestackedArgs); + PXOPT_ADD_MODE("-pendingcleanup_stacked", "", DETTOOL_MODE_PENDINGCLEANUP_STACKED, pendingcleanup_stackedArgs); + PXOPT_ADD_MODE("-donecleanup_stacked", "", DETTOOL_MODE_DONECLEANUP_STACKED, donecleanup_stackedArgs); + + PXOPT_ADD_MODE("-tonormalizedstat", "", DETTOOL_MODE_TONORMALIZEDSTAT, tonormalizedstatArgs); + PXOPT_ADD_MODE("-addnormalizedstat", "", DETTOOL_MODE_ADDNORMALIZEDSTAT, addnormstatArgs); + PXOPT_ADD_MODE("-normalizedstat", "", DETTOOL_MODE_NORMALIZEDSTAT, normalizedstatArgs); + PXOPT_ADD_MODE("-revertnormalizedstat", "", DETTOOL_MODE_REVERTNORMALIZEDSTAT, revertnormalizedstatArgs); + PXOPT_ADD_MODE("-updatenormalizedstat", "", DETTOOL_MODE_UPDATENORMALIZEDSTAT, updatenormalizedstatArgs); + PXOPT_ADD_MODE("-pendingcleanup_normalizedstat", "", DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDSTAT, pendingcleanup_normalizedstatArgs); + PXOPT_ADD_MODE("-donecleanup_normalizedstat", "", DETTOOL_MODE_DONECLEANUP_NORMALIZEDSTAT, donecleanup_normalizedstatArgs); + + PXOPT_ADD_MODE("-tonormalize", "", DETTOOL_MODE_TONORMALIZE, tonormalizeArgs); + PXOPT_ADD_MODE("-addnormalizedimfile", "", DETTOOL_MODE_ADDNORMALIZEDIMFILE,addnormalizedimfileArgs); + PXOPT_ADD_MODE("-normalizedimfile","", DETTOOL_MODE_NORMALIZEDIMFILE, normalizedimfileArgs); + PXOPT_ADD_MODE("-revertnormalizedimfile","", DETTOOL_MODE_REVERTNORMALIZEDIMFILE, revertnormalizedimfileArgs); + PXOPT_ADD_MODE("-updatenormalizedimfile","", DETTOOL_MODE_UPDATENORMALIZEDIMFILE, updatenormalizedimfileArgs); + PXOPT_ADD_MODE("-pendingcleanup_normalizedimfile", "", DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDIMFILE, pendingcleanup_normalizedimfileArgs); + PXOPT_ADD_MODE("-donecleanup_normalizedimfile", "", DETTOOL_MODE_DONECLEANUP_NORMALIZEDIMFILE, donecleanup_normalizedimfileArgs); + + PXOPT_ADD_MODE("-tonormalizedexp", "", DETTOOL_MODE_TONORMALIZEDEXP, tonormalizedexpArgs); + PXOPT_ADD_MODE("-addnormalizedexp", "", DETTOOL_MODE_ADDNORMALIZEDEXP, addnormalizedexpArgs); + PXOPT_ADD_MODE("-normalizedexp", "", DETTOOL_MODE_NORMALIZEDEXP, normalizedexpArgs); + PXOPT_ADD_MODE("-revertnormalizedexp","", DETTOOL_MODE_REVERTNORMALIZEDEXP, revertnormalizedexpArgs); + PXOPT_ADD_MODE("-updatenormalizedexp","", DETTOOL_MODE_UPDATENORMALIZEDEXP, updatenormalizedexpArgs); + PXOPT_ADD_MODE("-pendingcleanup_normalizedexp", "", DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDEXP, pendingcleanup_normalizedexpArgs); + PXOPT_ADD_MODE("-donecleanup_normalizedexp", "", DETTOOL_MODE_DONECLEANUP_NORMALIZEDEXP, donecleanup_normalizedexpArgs); + + PXOPT_ADD_MODE("-toresidimfile", "", DETTOOL_MODE_TORESIDIMFILE, toresidimfileArgs); + PXOPT_ADD_MODE("-addresidimfile", "", DETTOOL_MODE_ADDRESIDIMFILE, addresidimfileArgs); + PXOPT_ADD_MODE("-residimfile", "", DETTOOL_MODE_RESIDIMFILE, residimfileArgs); + PXOPT_ADD_MODE("-revertresidimfile", "", DETTOOL_MODE_REVERTRESIDIMFILE, revertresidimfileArgs); + PXOPT_ADD_MODE("-updateresidimfile", "", DETTOOL_MODE_UPDATERESIDIMFILE, updateresidimfileArgs); + PXOPT_ADD_MODE("-pendingcleanup_residimfile", "", DETTOOL_MODE_PENDINGCLEANUP_RESIDIMFILE, pendingcleanup_residimfileArgs); + PXOPT_ADD_MODE("-donecleanup_residimfile", "", DETTOOL_MODE_DONECLEANUP_RESIDIMFILE, donecleanup_residimfileArgs); + + PXOPT_ADD_MODE("-toresidexp", "", DETTOOL_MODE_TORESIDEXP, toresidexpArgs); + PXOPT_ADD_MODE("-addresidexp", "", DETTOOL_MODE_ADDRESIDEXP, addresidexpArgs); + PXOPT_ADD_MODE("-residexp", "", DETTOOL_MODE_RESIDEXP, residexpArgs); + PXOPT_ADD_MODE("-revertresidexp", "", DETTOOL_MODE_REVERTRESIDEXP, revertresidexpArgs); + PXOPT_ADD_MODE("-updateresidexp", "", DETTOOL_MODE_UPDATERESIDEXP, updateresidexpArgs); + PXOPT_ADD_MODE("-pendingcleanup_residexp", "", DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, pendingcleanup_residexpArgs); + PXOPT_ADD_MODE("-donecleanup_residexp", "", DETTOOL_MODE_DONECLEANUP_RESIDEXP, donecleanup_residexpArgs); + + PXOPT_ADD_MODE("-todetrunsummary", "", DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryArgs); + PXOPT_ADD_MODE("-adddetrunsummary", "", DETTOOL_MODE_ADDDETRUNSUMMARY,adddetrunsummaryArgs); + PXOPT_ADD_MODE("-detrunsummary", "", DETTOOL_MODE_DETRUNSUMMARY,detrunsummaryArgs); + PXOPT_ADD_MODE("-revertdetrunsummary", "", DETTOOL_MODE_REVERTDETRUNSUMMARY, revertdetrunsummaryArgs); + PXOPT_ADD_MODE("-updatedetrunsummary", "", DETTOOL_MODE_UPDATEDETRUNSUMMARY, updatedetrunsummaryArgs); + + PXOPT_ADD_MODE("-updatedetrun", "", DETTOOL_MODE_UPDATEDETRUN, updatedetrunArgs); + PXOPT_ADD_MODE("-rerun", "", DETTOOL_MODE_RERUN, rerunArgs); + PXOPT_ADD_MODE("-register_detrend", "", DETTOOL_MODE_REGISTER_DETREND, register_detrendArgs); + PXOPT_ADD_MODE("-register_detrend_imfile", "", DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_correction.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_correction.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_correction.c (revision 22235) @@ -0,0 +1,266 @@ +/* + * dettool_correction.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool makecorrectionMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(ref_iter, config->args, "-iteration", true, false); // required + PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); // required + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // optional + + // optional modifications + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + // build the needed where + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", ref_det_id); + psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", ref_iter); + + // select the detRun that matches + psArray *runs = detRunSelectRowObjects(config->dbh, where, 1); + psFree (where); + + if (!psArrayLength(runs)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + detRunRow *detRun = runs->data[0]; + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + char *use_workdir = (workdir) ? workdir : detRun->workdir; + char *use_reduction = (reduction) ? reduction : detRun->reduction; + char *use_label = (label) ? label : detRun->label; + + detRunInsert(config->dbh, + 0, // det_id + 0, // iteration + det_type, + "correction", // mode + "run", // state + detRun->filelevel, + use_workdir, + detRun->camera, + detRun->telescope, + detRun->exp_type, + use_reduction, + detRun->filter, + detRun->airmass_min, + detRun->airmass_max, + detRun->exp_time_min, + detRun->exp_time_max, + detRun->ccd_temp_min, + detRun->ccd_temp_max, + detRun->posang_min, + detRun->posang_max, + detRun->registered, + detRun->time_begin, + detRun->time_end, + detRun->use_begin, + detRun->use_end, + detRun->solang_min, + detRun->solang_max, + use_label, + detRun->det_id, // ref_det_id + detRun->iteration // ref_iter + ); + psFree(runs); + + // print the new detRun + psS64 new_det_id = psDBLastInsertID(config->dbh); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psMetadata *where_new = psMetadataAlloc(); + psMetadataAddS64(where_new, PS_LIST_TAIL, "det_id", 0, "==", new_det_id); + psArray *detRuns = psDBSelectRows(config->dbh, "detRun", where_new, 0); + psFree(where_new); + + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 "(this should not happen)", new_det_id); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +bool tocorrectimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tocorrectimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + // NOTE the SQL uses an intermediate table 'det1' for this query + psString whereClause = psDBGenerateWhereConditionSQL(where, "det1"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree (where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingCorrectImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool tocorrectexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tocorrectexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// NOTE : the command-line args are parsed by register_detrend_imfileMode +bool addcorrectimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // insert the row into detRegisterImfile + if (!register_detrend_imfileMode(config)) { + return false; + } + + // automatically stop completed 'correct' detRuns + psString query = pxDataGet("dettool_stop_completed_correct_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_detrunsummary.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_detrunsummary.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_detrunsummary.c (revision 22235) @@ -0,0 +1,366 @@ +/* + * dettool_detrunsummary.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row); + +/* The SQL returns a list of detrend runs (with detrend id, iteration and detrend type) which + * have completed all residexps. + */ + +bool todetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_todetrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRejectExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// this function is used to generate the detrunSummary entries used below +// XXX why is this a separate function? roll it back into adddetrunsummary? +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + // from row + psS64 det_id = psMetadataLookupS64(&status, row, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); + return false; + } + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); + + return detRunSummaryRowAlloc( + det_id, + iteration, + "full", + bg, + bg_stdev, + bg_mean_stdev, + accept, + code + ); +} + +// XXX data_state should be set to 'full' here +bool adddetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_BOOL(again, config->args, "-again", false); + + // The values supplied as arguments on the command (eg, -bg) are parsed + // by mdToDetRunSummary below. + // XXX why is there ever more than one? + + psString query = pxDataGet("dettool_find_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // XXX why is there ever more than one result here? + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detResidExp object + detRunSummaryRow *runSummary = mdToDetRunSummary(config, row); + if (!runSummary) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata to detResidExp"); + psFree(output); + return false; + } + // insert detResidExp object into the database + if (!detRunSummaryInsertObject(config->dbh, runSummary)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(runSummary); + psFree(output); + return false; + } + psFree(runSummary); + } + + psFree(output); + + // XXX this logic does not deal with the case of -code being set + // XXX it should be an error for -again and -code to both be set + if (again) { + if (!startNewIteration(config, det_id)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to start new iteration"); + return false; + } + } else { + // set detRun.state to stop + if (!setDetRunState(config, det_id, "stop")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set detRun.state"); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; +} + +bool detrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = pxDataGet("dettool_detrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "NULL"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detRunSummary.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detRunSummary.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawDetrendImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool revertdetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertdetrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRunSummary"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +// XXX need to add -data_state here +bool updatedetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); + PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + + if (accept && reject) { + psError(PS_ERR_UNKNOWN, true, "-accept and -reject are exclusive"); + return false; + } + + if (!(accept || reject)) { + psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required"); + return false; + } + + char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; + if (!p_psDBRunQuery(config->dbh, query, accept, det_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_normalizedexp.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_normalizedexp.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_normalizedexp.c (revision 22235) @@ -0,0 +1,389 @@ +/* + * dettool_normalizedexp.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool tonormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tonormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +bool addnormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0 + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + psString query = pxDataGet("dettool_tonormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX in some other places, we put the arguments in the .sql and supply them to RunQuery + psStringAppend(&query, " WHERE det_id = %" PRId64 " AND iteration = %d", det_id, iteration); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + psFree(output); + + // insert the new row into the detProcessedImfile table + if (!detNormalizedExpInsert( + config->dbh, + det_id, + iteration, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +bool normalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = pxDataGet("dettool_normalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detNormalizedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detNormalizedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + + +bool revertnormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertnormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updatenormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + + if (!setNormExpDataState(config, det_id, iteration, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_normalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedExp.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedExp.iteration", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_normalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupNormExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool donecleanup_normalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedExp.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedExp.iteration", "=="); + + psString query = pxDataGet("dettool_donecleanup_normalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_normalizedexp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_normalizedimfile.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_normalizedimfile.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_normalizedimfile.c (revision 22235) @@ -0,0 +1,372 @@ +/* + * dettool_normalizedimfile.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +// XXX this one is inconsistent - rename 'tonormalizedimfile' and change ipp_serial_detrend.pl and detrend.norm.pro +bool tonormalizeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tonormalize.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool addnormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // Required if code == 0 + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + if (!detNormalizedImfileInsert( + config->dbh, + det_id, + iteration, + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +bool normalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = pxDataGet("dettool_normalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detNormalizedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detNormalizedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool revertnormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertnormalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updatenormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + + if (!setNormImfileDataState(config, det_id, iteration, class_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_normalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detNormalizedImfile.det_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_normalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupNormImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool donecleanup_normalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detNormalizedImfile.det_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_normalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_normalizedimfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_normalizedstat.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_normalizedstat.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_normalizedstat.c (revision 22235) @@ -0,0 +1,352 @@ +/* + * dettool_normalizedstat.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool tonormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tonormalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool addnormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + // optional + PXOPT_LOOKUP_F32(norm, config->args, "-norm", false, false); + + // default + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + detNormalizedStatImfileRow *row = detNormalizedStatImfileRowAlloc + (det_id, + iteration, + class_id, + norm, + "full", + code); + + if (!detNormalizedStatImfileInsertObject(config->dbh, row)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +bool normalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_normalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "WHERE fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "WHERE fault = 0"); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool revertnormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertnormalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedStatImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updatenormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + + if (!setNormStatImfileDataState(config, det_id, iteration, class_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_normalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedStatImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedStatImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detNormalizedStatImfile.det_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_normalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupNormStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool donecleanup_normalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedStatImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedStatImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detNormalizedStatImfile.det_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_normalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_normalizedstat", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_processedexp.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_processedexp.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_processedexp.c (revision 22235) @@ -0,0 +1,398 @@ +/* + * dettool_processedexp.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool toprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_toprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, exp_id, recip, -bg, -bg_stdev + // are required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // Required if code == 0 + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F64(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F64(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + psString query = pxDataGet("dettool_addprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query, (long long) det_id, (long long) exp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + psFree(output); + + // create a new detProcessedImfile object + detProcessedExpRow *detRow = detProcessedExpRowAlloc( + det_id, + exp_id, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + code + ); + + // insert the new row into the detProcessedImfile table + if (!detProcessedExpInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM detProcessedExp"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detProcessedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detProcessedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detProcessedExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + if (!setProcessedExpDataState(config, det_id, exp_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// XXX SQL missing +bool donecleanup_processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanupProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_processedimfile.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_processedimfile.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_processedimfile.c (revision 22235) @@ -0,0 +1,433 @@ +/* + * dettool_processedimfile.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool toprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_toprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree (where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, exp_id, class_id, uri, recipe, -bg, -bg_stdev are required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // Required if code == 0 + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F64(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F64(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + // find the matching rawImfile by exp_id/class_id + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psArray *rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + psFree(where); + + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found "); + return false; + } + + // create a new detProcessedImfile object + detProcessedImfileRow *detRow = detProcessedImfileRowAlloc( + det_id, + exp_id, + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + code + ); + psFree(rawImfiles); + + // insert the new row into the detProcessedImfile table + if (!detProcessedImfileInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool hasWhere = false; + + PXOPT_LOOKUP_BOOL(included, config->args, "-included", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detProcessedImfile.det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "detProcessedImfile.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detProcessedImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-select_state", "detRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-select_mode", "detRun.mode", "=="); + + psString query = pxDataGet("dettool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + hasWhere = true; + } + psFree (where); + + // restrict search to included imfiles + if (included) { + if (hasWhere) { + psStringAppend(&query, " AND detInputExp.include = 1"); + } else { + psStringAppend(&query, " WHERE detInputExp.include = 1"); + } + hasWhere = true; + } + + if (hasWhere) { + psStringAppend(&query, " AND"); + } else { + psStringAppend(&query, " WHERE"); + } + + if (faulted) { + psStringAppend(&query, " %s", " detProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", " detProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updateprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + + if (!setProcessedImfileDataState(config, det_id, exp_id, class_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// XXX SQL missing +bool donecleanup_processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_processedimfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_residexp.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_residexp.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_residexp.c (revision 22235) @@ -0,0 +1,479 @@ +/* + * dettool_residexp.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +/* + The SQL returns a list of exposures for which all component class ids have had residuals + created. This list includes the detrend id, iteration, exposure id, detrend type, and + whether the exposure was included in the stack for this iteration. +*/ + +bool toresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_toresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +/* + The SQL returns a list of exposures for which all component class ids have had residuals + created. This list includes the detrend id, iteration, exposure id, detrend type, and + whether the exposure was included in the stack for this iteration. +*/ + +bool addresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // limit search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); // required + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0 + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(bg_skewness, config->args, "-bg_skewness", false, false); + PXOPT_LOOKUP_F64(bg_kurtosis, config->args, "-bg_kurtosis", false, false); + PXOPT_LOOKUP_F64(bin_stdev, config->args, "-bin_stdev", false, false); + PXOPT_LOOKUP_F64(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F64(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F64(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F64(fringe_resid_0, config->args, "-fringe_resid_0", false, false); + PXOPT_LOOKUP_F64(fringe_resid_1, config->args, "-fringe_resid_1", false, false); + PXOPT_LOOKUP_F64(fringe_resid_2, config->args, "-fringe_resid_2", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + + psString query = pxDataGet("dettool_toresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detResidImfile rows found"); + psFree(output); + return false; + } + psFree(output); + + if (!detResidExpInsert( + config->dbh, + det_id, + iteration, + exp_id, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + bg_skewness, + bg_kurtosis, + bin_stdev, + fringe_0, + fringe_1, + fringe_2, + fringe_resid_0, + fringe_resid_1, + fringe_resid_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + !reject, + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +bool residexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + + psString query = pxDataGet("dettool_residexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detResidExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detResidExp.fault = 0"); + } + + // list only accepted rows + // XXX the usage of this flag is unclear : -reject means "accepted"? + if (reject) { + psStringAppend(&query, " %s", "AND detResidExp.accept != 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool revertresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +bool updateresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + // find the values we're going to set + // copy everything but det_id, iteration, & exp_id from the args and + // remove the '-' prefix + psMetadata *set = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, set, "-recip", "recipe", "=="); + PXOPT_COPY_F64(config->args, set, "-bg", "bg", "=="); + PXOPT_COPY_F64(config->args, set, "-bg_stdev", "bg_stdev", "=="); + PXOPT_COPY_F64(config->args, set, "-bg_mean_stdev", "bg_mean_stdev", "=="); + PXOPT_COPY_STR(config->args, set, "-path_base", "path_base", "=="); + PXOPT_COPY_STR(config->args, set, "-data_state", "data_state", "=="); + + // double-check the allowed data_state values + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", false, false); + if (data_state) { + if (!isValidDataState (data_state)) return false; + } + + // this can't be PXOPT_ macro-ized as reject is !'d + if (!psMetadataAddBool(set, PS_LIST_TAIL, "accept", 0, "==", !reject)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item accept"); + psFree(set); + psFree(where); + return false; + } + + long changed = psDBUpdateRows(config->dbh, "detResidExp", where, set); + psFree(set); + psFree(where); + + if (changed < 0) { + psError(PS_ERR_UNKNOWN, false, "no rows were updated"); + return false; + } + + return true; +} + +bool pendingcleanup_residexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_residexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool donecleanup_residexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_residexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_residexp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_residimfile.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_residimfile.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_residimfile.c (revision 22235) @@ -0,0 +1,410 @@ +/* + * dettool_residimfile.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool toresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_toresidimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool addresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", true, false); // required + PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", true, false); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); // required + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // required + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); // Required if code == 0 + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0 + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(bg_skewness, config->args, "-bg_skewness", false, false); + PXOPT_LOOKUP_F64(bg_kurtosis, config->args, "-bg_kurtosis", false, false); + PXOPT_LOOKUP_F64(bin_stdev, config->args, "-bin_stdev", false, false); + PXOPT_LOOKUP_F64(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F64(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F64(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F64(fringe_resid_0, config->args, "-fringe_resid_0", false, false); + PXOPT_LOOKUP_F64(fringe_resid_1, config->args, "-fringe_resid_1", false, false); + PXOPT_LOOKUP_F64(fringe_resid_2, config->args, "-fringe_resid_2", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + if (!detResidImfileInsert( + config->dbh, + det_id, + iteration, + ref_det_id, + ref_iter, + exp_id, + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + bg_skewness, + bg_kurtosis, + bin_stdev, + fringe_0, + fringe_1, + fringe_2, + fringe_resid_0, + fringe_resid_1, + fringe_resid_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +bool residimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detResidImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detResidImfile.iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "detResidImfile.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detResidImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "detResidImfile.recipe", "=="); + PXOPT_COPY_STR(config->args, where, "-select_state", "detRun.state", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = pxDataGet("dettool_residimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + bool hasWhere = false; + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + hasWhere = true; + } + psFree(where); + + if (hasWhere) { + psStringAppend(&query, " AND"); + } else { + psStringAppend(&query, " WHERE"); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " detResidImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", " detResidImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool revertresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertresidimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updateresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + if (!setResidImfileDataState(config, det_id, iteration, exp_id, class_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_residimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_residimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool donecleanup_residimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_residimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_residimfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_stack.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_stack.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/dettool_stack.c (revision 22235) @@ -0,0 +1,457 @@ +/* + * dettool_stack.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +static psArray *searchRawImfiles(pxConfig *config); + +bool tostackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tostacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingStackedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// this is NOT a command-line mode : it is used by 'addstackedMode' +static psArray *searchRawImfiles(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + // select exp_ids from detInputExp matching det_id & iteration + // where query should be pre-generated + psArray *detInputExp = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree (where); + + if (!detInputExp) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return NULL; + } + + // generate where query with just the exp_ids + psMetadata *where_exp_ids = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(detInputExp); i++) { + detInputExpRow *row = detInputExp->data[i]; + if (!psMetadataAddS64(where_exp_ids, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", row->exp_id) + ) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(detInputExp); + psFree(where_exp_ids); + return NULL; + } + } + psFree(detInputExp); + + // select rawImfiles with matching exp_ids + psArray *rawImfiles = + rawImfileSelectRowObjects(config->dbh, where_exp_ids, 0); + psFree(where_exp_ids); + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} + +bool addstackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, iteration, class_id, uri, & recipe are required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // Required if code == 0 + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + + // correlate the class_id against the input exposure(s) + // searchRawImfiles defines a where clause to search by det_id and iteration + psArray *rawImfiles = searchRawImfiles(config); + + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "failed to get rawImfiles from db"); + return false; + } + + bool valid_class_id = false; + if (rawImfiles) { + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + if (!rawImfiles->data[i]) { + fprintf (stderr, "*"); + continue; + } + if (strcmp(class_id, ((rawImfileRow *)rawImfiles->data[i])->class_id) == 0) { + valid_class_id = true; + break; + } + } + psFree(rawImfiles); + } + + if (!valid_class_id) { + psError(PS_ERR_UNKNOWN, true, + "class_id can not be correlated with the input exposures"); + return false; + } + + // create a new detStackedImfile object + detStackedImfileRow *stackedImfile = detStackedImfileRowAlloc( + det_id, + iteration, + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + "full", + code + ); + + // insert the new row into the detProcessedImfile table + if (!detStackedImfileInsertObject(config->dbh, stackedImfile)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(stackedImfile); + return false; + } + + psFree(stackedImfile); + + return true; +} + +bool stackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_stacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detStackedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detStackedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool revertstackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertstacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updatestackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + + if (!setStackedImfileDataState(config, det_id, iteration, class_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_stackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detStackedImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detStackedImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detStackedImfile.class_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_stacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupStackedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// XXX SQL missing +bool donecleanup_stackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detStackedImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detStackedImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detStackedImfile.class_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_stacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_stacked", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/difftool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/difftool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/difftool.c (revision 22235) @@ -0,0 +1,1257 @@ +/* + * difftool.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "difftool.h" + +static bool definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputskyfileMode(pxConfig *config); +static bool inputskyfileMode(pxConfig *config); +static bool todiffskyfileMode(pxConfig *config); +static bool adddiffskyfileMode(pxConfig *config); +static bool diffskyfileMode(pxConfig *config); +static bool revertdiffskyfileMode(pxConfig *config); +static bool definepoprunMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupskyfileMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool updatediffskyfileMode(pxConfig *config); + +static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = difftoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DIFFTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(DIFFTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(DIFFTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileMode); + MODECASE(DIFFTOOL_MODE_INPUTSKYFILE, inputskyfileMode); + MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileMode); + MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileMode); + MODECASE(DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileMode); + MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode); + MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); + MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); + MODECASE(DIFFTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(DIFFTOOL_MODE_UPDATEDIFFSKYFILE, updatediffskyfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + + // default + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + label, + reduction, + workdir, + NULL, // dvodb + registered, + skycell_id, + tess_id + ); + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object"); + return true; + } + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + + // get the assigned diff_id + run->diff_id = psDBLastInsertID(config->dbh); + + if (!diffRunPrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(run); + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (state) { + // set detRun.state to state + return setdiffRunState(config, diff_id, state); + } + + return true; +} + + +static bool addinputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + + // optional + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false); + PXOPT_LOOKUP_STR(kind, config->args, "-kind", false, false); + + // defaults to false + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + + // must provide either stack_id or warp_id but not BOTH + if (!(stack_id || warp_id)) { + psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified"); + return false; + } + if (stack_id && warp_id) { + psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified"); + return false; + } + + // if a warp_id was provided we need to lookup the skycell_id and tess_id + // from the warpRun + psString skycell_id = NULL; + psString tess_id = NULL; + if (warp_id) { + if (!p_psDBRunQuery(config->dbh, "SELECT * from diffRun WHERE diff_id = %" PRId64, diff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "diff_id %" PRId64 " not found", diff_id); + psFree(output); + return false; + } + + diffRunRow *run = diffRunObjectFromMetadata(output->data[0]); + skycell_id = run->skycell_id; + tess_id = run->tess_id; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!diffInputSkyfileInsert(config->dbh, + diff_id, + template, + stack_id ? stack_id : PS_MAX_S64, // defined or NULL + warp_id ? warp_id : PS_MAX_S64, // defined or NULL + skycell_id, + tess_id, + kind + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, "SELECT count(diff_id) FROM diffRun JOIN diffInputSkyfile USING(diff_id) WHERE diff_id = %" PRId64, diff_id)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "diff_id %" PRId64 " not found", diff_id); + psFree(output); + return false; + } + + bool status; + psS32 count = psMetadataLookupS32(&status, output->data[0], "count(diff_id)"); + + if (count == 2) { + if (!setdiffRunState(config, diff_id, "new")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool inputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-kind", "kind", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + PXOPT_LOOKUP_BOOL(input, config->args, "-input", false); + + if (template && input) { + // User apparently wants both, which is the default behaviour + template = false; + input = false; + } + + // find all rawImfiles matching the default query + psString query = pxDataGet("difftool_inputskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = NULL; + if (psListLength(where->list)) { + whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause, "\n AND "); + } + psFree(where); + + // Add condition to get only templates or only inputs + { + psString templateClause = NULL; + if (template) { + psStringAppend(&templateClause, " %s", " template != 0"); + } else if (input) { + psStringAppend(&templateClause, " %s", " template = 0"); + } + if (templateClause) { + psStringAppend(&whereClause, "\n AND %s", templateClause); + } + psFree(templateClause); + } + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query, whereClause, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffInputSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool todiffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "diffRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("difftool_todiffskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool adddiffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // required + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (code == 0), false); + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_diff, config->args, "-dtime_diff", false, false); + PXOPT_LOOKUP_F32(dtime_match, config->args, "-dtime_match", false, false); + PXOPT_LOOKUP_F32(dtime_phot, config->args, "-dtime_phot", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_S32(stamps_num, config->args, "-stamps_num", false, false); + PXOPT_LOOKUP_F32(stamps_mean, config->args, "-stamps_mean", false, false); + PXOPT_LOOKUP_F32(stamps_rms, config->args, "-stamps_rms", false, false); + PXOPT_LOOKUP_F32(norm, config->args, "-norm", false, false); + PXOPT_LOOKUP_F32(bg_diff, config->args, "-bg_diff", false, false); + PXOPT_LOOKUP_F32(kernel_x, config->args, "-kernel_x", false, false); + PXOPT_LOOKUP_F32(kernel_y, config->args, "-kernel_y", false, false); + PXOPT_LOOKUP_F32(kernel_xx, config->args, "-kernel_xx", false, false); + PXOPT_LOOKUP_F32(kernel_xy, config->args, "-kernel_xy", false, false); + PXOPT_LOOKUP_F32(kernel_yy, config->args, "-kernel_yy", false, false); + PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!diffSkyfileInsert(config->dbh, + diff_id, + uri, + path_base, + bg, + bg_stdev, + stamps_num, + stamps_mean, + stamps_rms, + norm, + bg_diff, + kernel_x, + kernel_y, + kernel_xx, + kernel_xy, + kernel_yy, + sources, + dtime_diff, + dtime_match, + dtime_phot, + dtime_script, + hostname, + good_frac, + code + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!setdiffRunState(config, diff_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool diffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "diffSkyfile.fault", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("difftool_skyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertdiffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "diffRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + int num; // Number affected + + // Update state to 'new' + { + psString query = pxDataGet("difftool_revertdiffskyfile_update.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + num = psDBAffectedRows(config->dbh); + + if (num < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + + // Delete product + { + psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) != num) { + psError(PS_ERR_UNKNOWN, false, "Updated and deleted different number of entries!"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid diffRun state: %s", state); + return false; + } + + char *query = "UPDATE diffRun SET state = '%s' WHERE diff_id = %"PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, diff_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for diff_id %"PRId64, diff_id); + return false; + } + + return true; +} + +// Generate a single populated run +static bool populatedrun(psArray *list, // List of runs, to print + const char *workdir, // Working directory + const char *skycell_id, // Skycell identifier + const char *tess_id, // Tessellation identifier + const char *label, // label + const char *reduction, // reduction + psS64 input_warp_id, // Warp identifier for input image, PS_MAX_S64 for none + psS64 input_stack_id, // Stack identifier for input image, PS_MAX_S64 for none + psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none + psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none + pxConfig *config // Configuration + ) +{ + PS_ASSERT_STRING_NON_EMPTY(workdir, false); + PS_ASSERT_STRING_NON_EMPTY(skycell_id, false); + PS_ASSERT_STRING_NON_EMPTY(tess_id, false); + if ((input_warp_id == PS_MAX_S64 && input_stack_id == PS_MAX_S64) || + (input_warp_id != PS_MAX_S64 && input_stack_id != PS_MAX_S64)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One, and only one, input must be defined."); + return false; + } + if ((template_warp_id == PS_MAX_S64 && template_stack_id == PS_MAX_S64) || + (template_warp_id != PS_MAX_S64 && template_stack_id != PS_MAX_S64)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One, and only one, template must be defined."); + return false; + } + + // default + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + label, + reduction, + workdir, + NULL, // dvodb + registered, + skycell_id, + tess_id + ); + + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object"); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + + // get the assigned diff_id + run->diff_id = psDBLastInsertID(config->dbh); + + // Template + if (!diffInputSkyfileInsert(config->dbh, + run->diff_id, + true, + template_stack_id, + template_warp_id, + skycell_id, + tess_id, + NULL // kind + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // Input + if (!diffInputSkyfileInsert(config->dbh, + run->diff_id, + false, + input_stack_id, + input_warp_id, + skycell_id, + tess_id, + NULL // kind + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + char *query = "UPDATE diffRun SET state = 'new' WHERE diff_id = '%" PRId64 "'"; + if (!p_psDBRunQuery(config->dbh, query, run->diff_id)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to set state to run for diff_id %" PRId64, run->diff_id); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (list) { + psArrayAdd(list, list->n, run); + } + + psFree(run); + + return true; +} + + +static bool definepoprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); // required options + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required options + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_S64(template_warp_id, config->args, "-template_warp_id", false, false); + PXOPT_LOOKUP_S64(template_stack_id, config->args, "-template_stack_id", false, false); + PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false); + PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (template_stack_id && template_warp_id) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one template can be defined."); + return false; + } + if (!template_stack_id && !template_warp_id) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "No template has been defined (-template_stack_id or -template_warp_id)"); + return false; + } + + if (input_stack_id && input_warp_id) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one input can be defined."); + return false; + } + if (!input_stack_id && !input_warp_id) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "No input has been defined (-input_stack_id or -input_warp_id)"); + return false; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + + if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, + input_warp_id ? input_warp_id : PS_MAX_S64, + input_stack_id ? input_stack_id : PS_MAX_S64, + template_warp_id ? template_warp_id : PS_MAX_S64, + template_stack_id ? template_stack_id : PS_MAX_S64, + config)) { + psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun"); + psFree(list); + return false; + } + + if (!diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); + + return true; +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpsToDiff.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpsToDiff.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpsToDiff.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "warpsToDiff.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-stack_label", "stacksForDiff.stack_label", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_label", "warpsToDiff.warp_label", "=="); + PXOPT_COPY_STR(config->args, where, "-kind", "warpsToDiff.kind", "=="); + PXOPT_COPY_F32(config->args, where, "-good_frac", "warpsToDiff.good_frac", ">="); + + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); + + // find all things to queue + psString query = pxDataGet("difftool_definebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (newTemplates) { + // Warps that haven't been diffed OR warps that can take a newer template in the diff + psStringAppend(&query, " WHERE (diff_id IS NULL OR best_stack_id > current_stack_id)"); + } else { + // Only warps that haven't been diffed + psStringAppend(&query, " WHERE diff_id IS NULL"); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + long numGood = 0; // Number of good rows added + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Output row from query + + // Selected parameters + bool mdok; // Status of MD lookup + const char *skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id"); + if (!mdok || !skycell_id) { + psWarning("skycell_id not found --- ignoring row %ld", i); + continue; + } + const char *tess_id = psMetadataLookupStr(&mdok, row, "tess_id"); + if (!mdok || !tess_id) { + psWarning("tess_id not found --- ignoring row %ld", i); + continue; + } + psS64 stack_id = psMetadataLookupS64(&mdok, row, "best_stack_id"); + if (!mdok) { + psWarning("stack_id not found --- ignoring row %ld", i); + continue; + } + psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id"); + if (!mdok) { + psWarning("warp_id not found --- ignoring row %ld", i); + continue; + } + + if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id, PS_MAX_S64, PS_MAX_S64, + stack_id, config)) { + psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id, + warp_id, stack_id); + psErrorClear(); + continue; + } + + numGood++; + } + psFree(output); + + if (!diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("difftool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (diff_id) { + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("difftool_pendingcleanupskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffPendingCleanupSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("difftool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + return true; +} + +static bool updatediffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + + if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/difftool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/difftool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/difftool.h (revision 22235) @@ -0,0 +1,45 @@ +/* + * difftool.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DIFFTOOL_H +#define DIFFTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + DIFFTOOL_MODE_NONE = 0x0, + DIFFTOOL_MODE_DEFINERUN, + DIFFTOOL_MODE_UPDATERUN, + DIFFTOOL_MODE_ADDINPUTSKYFILE, + DIFFTOOL_MODE_INPUTSKYFILE, + DIFFTOOL_MODE_TODIFFSKYFILE, + DIFFTOOL_MODE_ADDDIFFSKYFILE, + DIFFTOOL_MODE_DIFFSKYFILE, + DIFFTOOL_MODE_REVERTDIFFSKYFILE, + DIFFTOOL_MODE_DEFINEPOPRUN, + DIFFTOOL_MODE_DEFINEBYQUERY, + DIFFTOOL_MODE_PENDINGCLEANUPRUN, + DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, + DIFFTOOL_MODE_DONECLEANUP, + DIFFTOOL_MODE_UPDATEDIFFSKYFILE, +} difftoolMode; + +pxConfig *difftoolConfig(pxConfig *config, int argc, char **argv); + +#endif // DIFFTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/difftoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/difftoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/difftoolConfig.c (revision 22235) @@ -0,0 +1,230 @@ +/* + * difftoolConfig.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "difftool.h" + +pxConfig *difftoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + + // -addinputskyfile + psMetadata *addinputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID", 0); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID", 0); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-kind", 0, "define kind", NULL); + psMetadataAddBool(addinputskyfileArgs, PS_LIST_TAIL, "-template", 0, "this sky cell file is the subtrahend", false); + + // -inputskyfile + psMetadata *inputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-kind", 0, "search by kind", NULL); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-template", 0, "find only subtrahend", false); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-input", 0, "find only minuend", false); + psMetadataAddU64(inputskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -todiffskyfile + psMetadata *todiffskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(todiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(todiffskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddU64(todiffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(todiffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -adddiffskyfile + psMetadata *adddiffskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define warp ID (required)", 0); + psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI of file", 0); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-dtime_diff", 0, "define elapsed processing time", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-dtime_match", 0, "define match processing time", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-dtime_phot", 0, "define photometry processing time", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS32(adddiffskyfileArgs, PS_LIST_TAIL, "-stamps_num", 0, "define subtraction stamp number", 0); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-stamps_mean", 0, "define subtraction stamp mean", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-stamps_rms", 0, "define subtraction stamp rms", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-norm", 0, "define normalisation", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-bg_diff", 0, "define background difference", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-kernel_x", 0, "define kernel x moment", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-kernel_y", 0, "define kernel y moment", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-kernel_xx", 0, "define kernel xx moment", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-kernel_xy", 0, "define kernel xy moment", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-kernel_yy", 0, "define kernel yy moment", NAN); + psMetadataAddS32(adddiffskyfileArgs, PS_LIST_TAIL, "-sources", 0, "define number of sources", 0); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); + + // -diffskyfile + psMetadata *diffskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by warp ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name", 0, "define exposure name", NULL); + psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-code", 0, "deifine fault code", 0); + + // -revertdiffskyfile + psMetadata *revertdiffskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertdiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddS16(revertdiffskyfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -definepoprun + psMetadata *definepoprunArgs = psMetadataAlloc(); + psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddTime(definepoprunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); + psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); + psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddBool(definepoprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_warp_id", 0, "define warp ID for template", 0); + psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_stack_id", 0, "define stack ID for template", 0); + psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-input_warp_id", 0, "define warp ID for input", 0); + psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-input_stack_id", 0, "define stack ID for input", 0); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-kind", 0, "search by kind", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupskyfile + psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -updatediffskyfile + psMetadata *updatediffskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(updatediffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); + psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", 0); + + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definerun", "", DIFFTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-updaterun", "", DIFFTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-addinputskyfile", "", DIFFTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs); + PXOPT_ADD_MODE("-inputskyfile", "", DIFFTOOL_MODE_INPUTSKYFILE, inputskyfileArgs); + PXOPT_ADD_MODE("-todiffskyfile", "", DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileArgs); + PXOPT_ADD_MODE("-adddiffskyfile", "", DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileArgs); + PXOPT_ADD_MODE("-diffskyfile", "", DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); + PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); + PXOPT_ADD_MODE("-definepoprun", "", DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunArgs); + PXOPT_ADD_MODE("-definebyquery", "", DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", DIFFTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-updatediffskyfile", "update fault code for a diffskyfile", DIFFTOOL_MODE_UPDATEDIFFSKYFILE, updatediffskyfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/fakemagic =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/fakemagic (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/fakemagic (revision 22235) @@ -0,0 +1,25 @@ +#!/usr/bin/env perl + +# Copyright (C) 2007 Joshua Hoblitt +# +# $Id: fakemagic,v 1.1 2007-11-05 23:29:54 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Getopt::Long qw( GetOptions :config auto_help auto_version pass_through ); +use Pod::Usage qw( pod2usage ); + +my ($output); +GetOptions( + 'output|o=s' => \$output, +) || pod2usage( 2 ); + +pod2usage( -msg => "Required options: --output ", -exitval => 2 ) + unless defined $output; + +open(my $fh, ">$output") or die "can't open file $output: $!"; +close($fh) or die "can't close file $output: $!"; Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/faketool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/faketool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/faketool.c (revision 22235) @@ -0,0 +1,1205 @@ +/* + * faketool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" + +#include "faketool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool revertprocessedimfileMode(pxConfig *config); +static bool updateprocessedimfileMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unmaskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupimfileMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool tocleanedimfileMode(pxConfig *config); +static bool tofullimfileMode(pxConfig *config); +static bool topurgedimfileMode(pxConfig *config); + +static bool fakeProcessedCompleteExp(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = faketoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(FAKETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(FAKETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(FAKETOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(FAKETOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(FAKETOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); + MODECASE(FAKETOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(FAKETOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); + MODECASE(FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + MODECASE(FAKETOOL_MODE_BLOCK, blockMode); + MODECASE(FAKETOOL_MODE_MASKED, maskedMode); + MODECASE(FAKETOOL_MODE_UNMASKED, unmaskedMode); + MODECASE(FAKETOOL_MODE_UNBLOCK, unblockMode); + MODECASE(FAKETOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(FAKETOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileMode); + MODECASE(FAKETOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(FAKETOOL_MODE_TOCLEANEDIMFILE, tocleanedimfileMode); + MODECASE(FAKETOOL_MODE_TOFULLIMFILE, tofullimfileMode); + MODECASE(FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + + // default + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("faketool_find_camrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (pretend) { + // then stop before running the query + fprintf(stderr, "%s\n", query); + psFree(query); + return true; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of cam_ids + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 cam_id = psMetadataLookupS64(&status, md, "cam_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for cam_id"); + psFree(output); + return false; + } + + // queue the exp + if (!pxfakeQueueByCamID(config, cam_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue cam_id: %" PRId64, cam_id); + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + where = NULL; + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set fakeRun.state to state + if (!pxfakeRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set fakeRun.label to label + if (!pxfakeRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + + psString query = pxDataGet("faketool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakePendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + + psString query = pxDataGet("faketool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakePendingImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // fake_id, ext_tag, class_id are required + PXOPT_LOOKUP_S64(fake_id, config->args, "-fake_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + + PXOPT_LOOKUP_F32(dtime_fake, config->args, "-dtime_fake", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fakeProcessedImfileInsert(config->dbh, + fake_id, + exp_id, + class_id, + uri, + dtime_fake, + dtime_script, + hostname, + path_base, + "full", + code, + NULL // epoch + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX I've decided to make the transaction cover the Exp migration as + // well. Otherwise, if the last imfile in an exp is moved and the exp + // migration fails then the data base is left in a satuation where the exp + // migration can't happen. + + if (!fakeProcessedCompleteExp(config)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + + psString query = pxDataGet("faketool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "fakeProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND fakeProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND fakeProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakeProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("faketool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool updateprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + if (!pxSetFaultCode(config->dbh, "fakeProcessedImfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree(where); + return false; + } + psFree(where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!fakeMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM fakeMask"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakeMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// return the list of labels which are NOT blocked +static bool unmaskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + psString query = pxDataGet("faketool_unmasked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, "fakeUnmask"); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakeUnmask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM fakeMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("faketool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakePendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(fake_id, config->args, "-fake_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (fake_id) { + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("faketool_pendingcleanupimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakePendingCleanupImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("faketool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakeDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool fakeProcessedCompleteExp(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // look for completed fakePendingExp + // migrate them to fakeProccessedExp & camPendingExp + psString query = pxDataGet("faketool_completely_processed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + fakeRunRow *fakeRun = fakeRunObjectFromMetadata(row); + // set fakeRun.state to 'full' + if (!pxfakeRunSetState(config, fakeRun->fake_id, "full")) { + psError(PS_ERR_UNKNOWN, false, "failed to change fakeRun.state for fake_id: %" PRId64, fakeRun->fake_id); + psFree(fakeRun); + psFree(output); + return false; + } + + // should we stop here or proceed on to the cam stage? + // NULL for end_stage means go as far as possible + if ((fakeRun->end_stage && psStrcasestr(fakeRun->end_stage, "fake")) || !fakeRun->tess_id) { + psFree(fakeRun); + continue; + } + // else continue on... + + // camQueueFakeID() can only be run after fakeRun.state has been set to + // stop + if (!pxwarpQueueByFakeID(config, + fakeRun->fake_id, + fakeRun->workdir, + fakeRun->label, + fakeRun->dvodb, + fakeRun->tess_id, + fakeRun->end_stage + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue camPendingExp"); + psFree(fakeRun); + psFree(output); + return false; + } + psFree(fakeRun); + } + + psFree(output); + + return true; +} + +// update fakeProcessedImfile.data_state to given value. +// afterwards, if all imfiles in the exposure have the new state, update the state for the exposure as well +// shared code for the modes -tocleanedimfile -tofullimfile -topurgedimfile + +static bool change_imfile_data_state(pxConfig *config, psString data_state, psString run_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // fake_id, class_id are required + PXOPT_LOOKUP_S64(fake_id, config->args, "-fake_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + psString query = pxDataGet("faketool_change_imfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // note only updates if fakeRun.state = run_state + if (!p_psDBRunQuery(config->dbh, query, data_state, fake_id, class_id, run_state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + query = pxDataGet("faketool_change_exp_state.sql"); + if (!p_psDBRunQuery(config->dbh, query, data_state, fake_id, data_state)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool tocleanedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "cleaned", "goto_cleaned"); +} +static bool tofullimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "full", "update"); +} +static bool topurgedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "purged", "goto_purged"); +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/faketool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/faketool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/faketool.h (revision 22235) @@ -0,0 +1,50 @@ +/* + * faketool.h + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef FAKETOOL_H +#define FAKETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + FAKETOOL_MODE_NONE = 0x0, + FAKETOOL_MODE_DEFINEBYQUERY, + FAKETOOL_MODE_UPDATERUN, + FAKETOOL_MODE_PENDINGIMFILE, + FAKETOOL_MODE_PENDINGEXP, + FAKETOOL_MODE_ADDPROCESSEDIMFILE, + FAKETOOL_MODE_PROCESSEDIMFILE, + FAKETOOL_MODE_REVERTPROCESSEDIMFILE, + FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, + FAKETOOL_MODE_BLOCK, + FAKETOOL_MODE_MASKED, + FAKETOOL_MODE_UNMASKED, + FAKETOOL_MODE_UNBLOCK, + FAKETOOL_MODE_RETRYPROCESSEDIMFILE, + FAKETOOL_MODE_PENDINGCLEANUPRUN, + FAKETOOL_MODE_PENDINGCLEANUPIMFILE, + FAKETOOL_MODE_DONECLEANUP, + FAKETOOL_MODE_TOCLEANEDIMFILE, + FAKETOOL_MODE_TOFULLIMFILE, + FAKETOOL_MODE_TOPURGEDIMFILE, +} FAKETOOLMode; + +pxConfig *faketoolConfig(pxConfig *config, int argc, char **argv); + +#endif // FAKETOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/faketoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/faketoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/faketoolConfig.c (revision 22235) @@ -0,0 +1,341 @@ +/* + * faketoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "faketool.h" + +pxConfig *faketoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *queueArgs = psMetadataAlloc(); + // XXX need to allow multiple exp_ids + psMetadataAddS64(queueArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chiptool ID", 0); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0, "search by camera of interest", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-filter", 0, "search by filter of interest", NULL); + psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "define fake ID (required)", 0); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID (required)", 0); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri", 0, "define URL", NULL); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-dtime_fake", 0, "define elapsed time for detrend (seconds)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "define fake ID", 0); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-inst", 0, "define camera of interest", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-filter", 0, "define filter of interest", NULL); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updateprocessedimfile + psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", 0); + + // -revertprocessedimfile + psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(revertprocessedimfileArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(revertprocessedimfileArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ccd_temp_min",0, "define min ccd tempature", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ccd_temp_max",0, "define max ccd tempature", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + + psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddStr(maskedArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(maskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unmasked + psMetadata *unmaskedArgs = psMetadataAlloc(); + psMetadataAddStr(unmaskedArgs, PS_LIST_TAIL, "-label", 0, "restrict to specified label", NULL); + psMetadataAddBool(unmaskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(unmaskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupimfile + psMetadata *pendingcleanupimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by chip ID", 0); + psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -tocleanedimfile + psMetadata *tocleanedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "fake ID to update", 0); + psMetadataAddStr(tocleanedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -tofullimfile + psMetadata *tofullimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "fake ID to update", 0); + psMetadataAddStr(tofullimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -topurgedimfile + psMetadata *topurgedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(topurgedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "fake ID to update", 0); + psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from raw exposures", FAKETOOL_MODE_DEFINEBYQUERY, queueArgs); + PXOPT_ADD_MODE("-updaterun", "change fake run properties", FAKETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exposures", FAKETOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-pendingimfile", "show pending imfiles", FAKETOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-addprocessedimfile", "add a processed imfile", FAKETOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXOPT_ADD_MODE("-processedimfile", "show processed imfiles", FAKETOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); + PXOPT_ADD_MODE("-revertprocessedimfile", "undo a processed imfile", FAKETOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); + PXOPT_ADD_MODE("-block", "set a label block", FAKETOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", FAKETOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unmasked", "", FAKETOOL_MODE_UNMASKED, unmaskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", FAKETOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", FAKETOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupimfile", "show runs that need to be cleaned up", FAKETOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", FAKETOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-tocleanedimfile", "set imfile state to cleaned", FAKETOOL_MODE_TOCLEANEDIMFILE, tocleanedimfileArgs); + PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", FAKETOOL_MODE_TOFULLIMFILE, tofullimfileArgs); + PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/flatcorr.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/flatcorr.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/flatcorr.c (revision 22235) @@ -0,0 +1,735 @@ +/* + * flatcorr.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * Copyright (C) 2008 Eugene Magnier + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include + +#include "pxtools.h" +#include "pxchip.h" +#include "flatcorr.h" + +static bool definebyqueryMode(pxConfig *config); +static bool definerunMode(pxConfig *config); +static bool addchipMode(pxConfig *config); +static bool addcameraMode(pxConfig *config); +static bool dropchipMode(pxConfig *config); +static bool dropcameraMode(pxConfig *config); +static bool pendingprocessMode(pxConfig *config); +static bool addprocessMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool inputexpMode(pxConfig *config); +static bool inputimfileMode(pxConfig *config); + +static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = flatcorrConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(FLATCORR_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(FLATCORR_MODE_DEFINERUN, definerunMode); + MODECASE(FLATCORR_MODE_ADDCHIP, addchipMode); + MODECASE(FLATCORR_MODE_ADDCAMERA, addcameraMode); + MODECASE(FLATCORR_MODE_DROPCHIP, dropchipMode); + MODECASE(FLATCORR_MODE_DROPCAMERA, dropcameraMode); + MODECASE(FLATCORR_MODE_PENDINGPROCESS, pendingprocessMode); + MODECASE(FLATCORR_MODE_ADDPROCESS, addprocessMode); + MODECASE(FLATCORR_MODE_UPDATERUN, updaterunMode); + MODECASE(FLATCORR_MODE_INPUTEXP, inputexpMode); + MODECASE(FLATCORR_MODE_INPUTIMFILE, inputimfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + // require the camera to be defined: this analysis does not make sense + // across multiple cameras + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); + + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-set_filter", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(region, config->args, "-set_region", false, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("chiptool_find_rawexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + for (long i = 0; i < psArrayLength(output); i++) { + // negative simple so the default is true + if (!ippdbPrintMetadataRaw(stdout, output->data[i], !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // create a new flatcorrRun + if (!flatcorrRunInsert( + config->dbh, + 0, // corr_id + det_type, + dvodb, + camera, + telescope, + NULL, + filter, + "reg", // state + workdir, + label, + reduction, + region, + NULL, + 0 + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // figure out the ID of the flatcorrRun we just created + psS64 corr_id = psDBLastInsertID(config->dbh); + + // loop over our list of exp_ids + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + if (!status) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psFree(output); + return false; + } + + // queue the exp : force this to stop at the chip stage + psS64 chip_id = pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, "chip"); + if (!chip_id) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue exp_id: %" PRId64, exp_id); + psFree(output); + return false; + } + + // add a flatcorrChipLink to the flatcorr Run we just created (initial state has include = TRUE) + if (!flatcorrChipLinkInsert(config->dbh, corr_id, chip_id, 1)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + } + psFree(output); + + // set the flatcorrRun to a state of 'new' + if (!setflatcorrRunState(config, corr_id, "new")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set run state"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // require the camera to be defined: this analysis does not make sense across multiple + // cameras + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); + + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-set_filter", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(region, config->args, "-set_region", false, false); + // XXX probably should make the region in -set_region match ra_min, ra_max, etc + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // create a new flatcorrRun + flatcorrRunRow *row = flatcorrRunRowAlloc( + 0, // corr_id + det_type, + dvodb, + camera, + telescope, + NULL, + filter, + "reg", // state + workdir, + label, + reduction, + region, + NULL, // hostname + 0 // fault + ); + + // create a new flatcorrRun + if (!flatcorrRunInsertObject(config->dbh,row)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // figure out the ID of the flatcorrRun we just created + psS64 corr_id = psDBLastInsertID(config->dbh); + row->corr_id = corr_id; + + flatcorrRunPrintObject (stdout, row, !simple); + return true; +} + +static bool addchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + + // add a flatcorrChipLink (initial state has include = TRUE) + if (!flatcorrChipLinkInsert(config->dbh, corr_id, chip_id, 1)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool dropchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + + // UPDATE flatcorrChipLink set include = 0 where corr_id = %lld AND chip_id = %lld + psString query = pxDataGet("flatcorr_dropchip.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query, (long long) corr_id, (long long) chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool addcameraMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psString query = pxDataGet("flatcorr_chiprundone.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "flatcorr_addcamera", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of chipRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 corr_id = psMetadataLookupS64(&status, md, "corr_id"); + if (!status) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for corr_id"); + psFree(output); + return false; + } + + chipRunRow *row = chipRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxcamQueueByChipID( + config, + row->chip_id, + row->workdir, + row->label, + row->reduction, + row->expgroup, + row->dvodb, + row->tess_id, + "camera")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->chip_id); + psFree(row); + psFree(output); + return false; + } + + // figure out the ID of the flatcorrRun we just created + psS64 cam_id = psDBLastInsertID(config->dbh); + + // add the camRun entry to the flatcorrCamLink table (include is TRUE) + if (!flatcorrCamLinkInsert(config->dbh, corr_id, row->chip_id, cam_id, 1)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return false; +} + +static bool dropcameraMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); + + // UPDATE flatcorrCamLink set include = 0 where corr_id = %lld AND cam_id = %lld + psString query = pxDataGet("flatcorr_dropcamera.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query, (long long) corr_id, (long long) cam_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool pendingprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + + psString query = pxDataGet("flatcorr_pendingprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + return true; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool inputexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + + char *query = psStringCopy ("SELECT * FROM flatcorrChipLink WHERE corr_id = %" PRId64); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query, corr_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + return true; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool inputimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipProcessedImfile.chip_id", "=="); + + psString query = pxDataGet("flatcorr_inputimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query, chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + return true; +} + +// XXX need a fault state +static bool addprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + char *query = "UPDATE flatcorrRun SET state = 'full', hostname = '%s', fault = '%hd' WHERE corr_id = %" PRId64; + + if (!p_psDBRunQuery(config->dbh, query, hostname, code, corr_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id); + return false; + } + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (!setflatcorrRunState(config, corr_id, state)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set run state"); + return false; + } + + return true; +} + +static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!strcmp(state, "reg") && + !strcmp(state, "new") && + !strcmp(state, "full")) + { + psError(PS_ERR_UNKNOWN, false, "invalid state: %s", state); + return false; + } + + char *query = "UPDATE flatcorrRun SET state = '%s' WHERE corr_id = %" PRId64; + + if (!p_psDBRunQuery(config->dbh, query, state, corr_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id); + return false; + } + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/flatcorr.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/flatcorr.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/flatcorr.h (revision 22235) @@ -0,0 +1,42 @@ +/* + * flatcorr.h + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef FLATCORR_H +#define FLATCORR_H 1 + +#include "pxtools.h" + +typedef enum { + FLATCORR_MODE_NONE = PXTOOL_MODE_NONE, + FLATCORR_MODE_DEFINEBYQUERY, + FLATCORR_MODE_DEFINERUN, + FLATCORR_MODE_ADDCHIP, + FLATCORR_MODE_ADDCAMERA, + FLATCORR_MODE_DROPCHIP, + FLATCORR_MODE_DROPCAMERA, + FLATCORR_MODE_PENDINGPROCESS, + FLATCORR_MODE_ADDPROCESS, + FLATCORR_MODE_UPDATERUN, + FLATCORR_MODE_INPUTEXP, + FLATCORR_MODE_INPUTIMFILE +} flatcorrMode; + +pxConfig *flatcorrConfig(pxConfig *config, int argc, char **argv); + +#endif // FLATCORR_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/flatcorrConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/flatcorrConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/flatcorrConfig.c (revision 22235) @@ -0,0 +1,166 @@ +/* + * flatcorrConfig.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "flatcorr.h" + +pxConfig *flatcorrConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + // XXX need to allow multiple exp_ids + pxchipSetSearchArgs (definebyqueryArgs); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-det_type", 0, "define detrend type to be generated", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction",0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_filter", 0, "define filter", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessalation", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_region", 0, "define region", NULL); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-det_type", 0, "define detrend type to be generated", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_filter", 0, "define filter", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessalation", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_region", 0, "define region", NULL); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addchip + psMetadata *addchipArgs = psMetadataAlloc(); + psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-corr_id", 0, "define Flat Correction ID (required)", 0); + psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-chip_id", 0, "define Chip ID (required)", 0); + + // -dropchip + psMetadata *dropchipArgs = psMetadataAlloc(); + psMetadataAddS64(dropchipArgs, PS_LIST_TAIL, "-corr_id", 0, "define Flat Correction ID (required)", 0); + psMetadataAddS64(dropchipArgs, PS_LIST_TAIL, "-chip_id", 0, "define Chip ID (required)", 0); + + // -addcamera + psMetadata *addcameraArgs = psMetadataAlloc(); + psMetadataAddU64 (addcameraArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(addcameraArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(addcameraArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + + // -dropcamera + psMetadata *dropcameraArgs = psMetadataAlloc(); + psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-corr_id", 0, "define Flat Correction ID (required)", 0); + psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-cam_id", 0, "define Camera ID (required)", 0); + + // -pendingprocess + psMetadata *pendingprocessArgs = psMetadataAlloc(); + psMetadataAddU64 (pendingprocessArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingprocessArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocess (XXX need to add fault and stats) + psMetadata *addprocessArgs = psMetadataAlloc(); + psMetadataAddS64 (addprocessArgs, PS_LIST_TAIL, "-corr_id", 0, "add complete run for specified corr_id (required)", 0); + psMetadataAddStr (addprocessArgs, PS_LIST_TAIL, "-hostname", 0, "set hostname", NULL); + psMetadataAddS16 (addprocessArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-corr_id", 0, "define correction id (required)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + + // -inputexp + psMetadata *inputexpArgs = psMetadataAlloc(); + psMetadataAddS64(inputexpArgs, PS_LIST_TAIL, "-corr_id", 0, "define correction id (required)", 0); + psMetadataAddBool(inputexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(inputexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -inputimfile + psMetadata *inputimfileArgs = psMetadataAlloc(); + psMetadataAddS64(inputimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip id (required)", 0); + psMetadataAddBool(inputimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(inputimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create a new, populated flat correction run", FLATCORR_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "create a new, empty flat correction run", FLATCORR_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-addchip", "add a chip to a flat correction run", FLATCORR_MODE_ADDCHIP, addchipArgs); + PXOPT_ADD_MODE("-addcamera", "migrate completed chips to camera stage analysis", FLATCORR_MODE_ADDCAMERA, addcameraArgs); + PXOPT_ADD_MODE("-dropchip", "drop a chip from a flat correction run", FLATCORR_MODE_DROPCHIP, dropchipArgs); + PXOPT_ADD_MODE("-dropcamera", "drop an exposure (camera stage analysis)", FLATCORR_MODE_DROPCAMERA, dropcameraArgs); + PXOPT_ADD_MODE("-pendingprocess", "show flat correction runs needing to be processed", FLATCORR_MODE_PENDINGPROCESS, pendingprocessArgs); + PXOPT_ADD_MODE("-addprocess", "report completed flat correction analysis", FLATCORR_MODE_ADDPROCESS, addprocessArgs); + PXOPT_ADD_MODE("-updaterun", "change a flat calibration run's state", FLATCORR_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-inputexp", "list exposures for a correction run", FLATCORR_MODE_INPUTEXP, inputexpArgs); + PXOPT_ADD_MODE("-inputimfile", "list imfiles for a chip run", FLATCORR_MODE_INPUTIMFILE, inputimfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/magictool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/magictool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/magictool.c (revision 22235) @@ -0,0 +1,1278 @@ +/* + * magictool.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "magictool.h" + +static bool definebyqueryMode(pxConfig *config); +static psS64 definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputskyfileMode(pxConfig *config); +static bool inputskyfileMode(pxConfig *config); +static bool totreeMode(pxConfig *config); +static bool inputtreeMode(pxConfig *config); +static bool reverttreeMode(pxConfig *config); +static bool toprocessMode(pxConfig *config); +static bool addresultMode(pxConfig *config); +static bool revertnodeMode(pxConfig *config); +static bool inputsMode(pxConfig *config); +static bool tomaskMode(pxConfig *config); +static bool addmaskMode(pxConfig *config); +static bool revertmaskMode(pxConfig *config); +static bool maskMode(pxConfig *config); + +static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state); +static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = magictoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(MAGICTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(MAGICTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(MAGICTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(MAGICTOOL_MODE_ADDINPUTSKYFILE,addinputskyfileMode); + MODECASE(MAGICTOOL_MODE_INPUTSKYFILE, inputskyfileMode); + MODECASE(MAGICTOOL_MODE_TOTREE, totreeMode); + MODECASE(MAGICTOOL_MODE_INPUTTREE, inputtreeMode); + MODECASE(MAGICTOOL_MODE_REVERTTREE, reverttreeMode); + MODECASE(MAGICTOOL_MODE_TOPROCESS, toprocessMode); + MODECASE(MAGICTOOL_MODE_ADDRESULT, addresultMode); + MODECASE(MAGICTOOL_MODE_REVERTNODE, revertnodeMode); + MODECASE(MAGICTOOL_MODE_INPUTS, inputsMode); + MODECASE(MAGICTOOL_MODE_TOMASK, tomaskMode); + MODECASE(MAGICTOOL_MODE_ADDMASK, addmaskMode); + MODECASE(MAGICTOOL_MODE_REVERTMASK, revertmaskMode); + MODECASE(MAGICTOOL_MODE_MASK, maskMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // Required + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + + // Optional + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // Create temporary table of the best diffs + { + psString query = pxDataGet("magictool_definebyquery_temp_create.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + } + + // Insert list of best diffs into temporary table + { + psString query = pxDataGet("magictool_definebyquery_temp_insert.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psString groupby = pxDataGet("magictool_definebyquery_temp_insert_groupby.sql"); + if (!groupby) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(query); + return false; + } + + psStringAppend(&query, " %s", groupby); + psFree(groupby); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + } + + // Get list of exposures ready to magic + { + psString query = pxDataGet("magictool_definebyquery_select_part1.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + } + + psString part2 = pxDataGet("magictool_definebyquery_select_part2.sql"); + if (!part2) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psStringAppend(&query, " %s", part2); + psFree(part2); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + } + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; // Row of interest + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier + + // create a new magicRun for this group + magicRunRow *run = magicRunRowAlloc(0, exp_id, "run", workdir, "dirty", label, dvodb, registered, 0); + if (!run) { + psAbort("failed to alloc magicRun object"); + } + + if (!magicRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + psFree(insert); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier + run->magic_id = magic_id; + + psArrayAdd(list, list->n, run); + psFree(run); + + // Create a suitable insertion query for this run + psString thisInsert = psStringCopy(insert); + { + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, magic_id); + psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@"); + psFree(idString); + } + { + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, exp_id); + psStringSubstitute(&thisInsert, idString, "@EXP_ID@"); + psFree(idString); + } + + if (!p_psDBRunQuery(config->dbh, thisInsert, magic_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(thisInsert); + psFree(insert); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(thisInsert); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(output); + + if (!magicRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + + psFree(list); + + return true; +} + +static psS64 definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + magicRunRow *run = magicRunRowAlloc( + 0, // ID + exp_id, + "reg", // state + workdir, + "dirty", // workdir_state + label, + dvodb, + registered, + 0 + ); + + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object"); + return false; + } + if (!magicRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return false; + } + + // get the assigned warp_id + psS64 magic_id = psDBLastInsertID(config->dbh); + run->magic_id = magic_id; + + if (!magicRunPrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(run); + + return magic_id; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (state) { + // set detRun.state to state + return setmagicRunState(config, magic_id, state); + } + + return true; +} + + +static bool addinputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(node, config->args, "-node", true, false); + + magicInputSkyfileInsert( + config->dbh, + magic_id, + diff_id, + node + ); + + return true; +} + + +static bool inputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("magictool_inputskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "magicInputSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicInputSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool totreeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magictool_totree.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "totree", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool inputtreeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + + // Optional values + PXOPT_LOOKUP_STR(dep_file, config->args, "-dep_file", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (code > 0) { + char *query = "UPDATE magicRun SET fault = %d, state = 'stop' WHERE magic_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, code, magic_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to set fault for magic_id %" PRId64, magic_id); + return false; + } + return true; + } + + if (!parseAndInsertNodeDeps(config, magic_id, dep_file)) { + psError(PS_ERR_UNKNOWN, false, "failed to parse file"); + return false; + } + + return true; +} + +static bool reverttreeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'run' WHERE fault != 0"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + return false; + } + return true; +} + + +static bool inputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magictool_inputs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicNode", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +bool findBaseNodes(void *arg, pxNode *node) +{ + bool status = false; + psS64 done = psMetadataLookupS64(&status, node->data, "done"); + if (!status) { + psAbort("failed to lookup value for done column"); + } + + if ((!pxNodeHasChildren(node)) && (!done)) { + // if this node has no child and it's not 'done', then push it's data + // onto the void *array + psArrayAdd((psArray *)arg, 0, node->data); + return false; + } + + return true; +} + + +bool findReadyNodes(void *arg, pxNode *node) +{ + if (!node) { + // It's not there --- must have failed + return false; + } + + if (!node->data) { + // It's a leaf node, not done + return false; + } + + if (psMetadataLookupBool(NULL, node->data, "done")) { + // It's already done + return true; + } + + if (pxNodeHasChildren(node)) { + psListIterator *iter = psListIteratorAlloc(node->children, 0, false); + psMetadata *work = psMetadataCopy(NULL, node->data); + psMetadataRemoveKey(work, "dep"); + psMetadataRemoveKey(work, "done"); + pxNode *child = NULL; + while ((child = psListGetAndIncrement(iter))) { + psMetadata *data = child->data; + if (!data) { + // Child is a leaf node, not done + psFree(iter); + psFree(work); + return false; + } + + bool status = false; + psS32 done = psMetadataLookupS32(&status, data, "done"); + if (!status) { + psAbort("failed to lookup value for done column"); + } + psS16 bad = psMetadataLookupS16(&status, data, "bad"); + if (!status) { + psAbort("failed to lookup value for bad column"); + } + + if (!done || bad) { + // if a child isn't "done", give up on this node and continue + // to crawl the tree + psFree(iter); + psFree(work); + return true; + } + } + psFree(iter); + // if all this nodes children are done, then push it's data onto the + // void *array + psArrayAdd((psArray *)arg, 0, work); + psFree(work); + return false; + } + + return true; +} + + +static bool toprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magictool_toprocess_inputs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = NULL; + if (psListLength(where->list)) { + whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + // psFree(output); + // return true; + } + + // look for tree nodes that need to be processed + query = pxDataGet("magictool_toprocess_tree.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (whereClause) { + psStringAppend(&query, " %s", whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *magicTree = p_psDBFetchResult(config->dbh); + if (!magicTree) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(magicTree)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(magicTree); + return true; + } + + psHash *forest = psHashAlloc(psArrayLength(magicTree)); + + // convert the array of metadata into a pxTree structure + for (long i = 0; i < psArrayLength(magicTree); i++) { + bool status; + psString node = psMetadataLookupStr(&status, magicTree->data[i], "node"); + if (!status) { + psAbort("failed to lookup value for node column"); + } + + psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep"); + if (!status) { + psAbort("failed to lookup value for dep column"); + } + + pxTreeBuilder(forest, node, dep, magicTree->data[i]); + + } + psFree(magicTree); + + // find the root of the tree + pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root")); + psFree(forest); + // pxTreePrint(stdout, root); + + // crawl through the tree and looking for nodes with children that are all + // "done" + pxTreeCrawl(root, findReadyNodes, output); + psFree(root); + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + psFree(whereClause); + + return true; +} + + +static bool addresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + PXOPT_LOOKUP_STR(node, config->args, "-node", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!magicNodeResultInsert(config->dbh, + magic_id, + node, + uri, + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool revertnodeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = psStringCopy("DELETE FROM magicNodeResult WHERE fault != 0"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + return false; + } + return true; +} + + +static bool tomaskMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magictool_tomask.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "tomask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addmaskMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_S32(streaks, config->args, "-streaks", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!magicMaskInsert(config->dbh, + magic_id, + uri, + streaks, + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // Set the magicRun state + psString query = pxDataGet("magictool_addmask.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // manually add constraint + psStringAppend(&query, " AND magic_id = %" PRId64, magic_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool revertmaskMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // Set to "run" + { + psString query = psStringCopy("UPDATE magicRun JOIN magicMask USING(magic_id) " + "SET magicRun.state = 'run' WHERE magicMask.fault != 0"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "magicMask"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psFree(query); + } + + // Delete failed attempt at mask + { + psString query = psStringCopy("DELETE FROM magicMask WHERE fault != 0"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "magicMask"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psFree(query); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(where); + + return true; +} + + +static bool maskMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magictool_mask.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid magicRun state: %s", state); + return false; + } + + char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, magic_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for magic_id %" PRId64, magic_id); + return false; + } + + return true; +} + +static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename) +{ + unsigned int nFail = 0; + psMetadata *deps = psMetadataConfigRead(NULL, &nFail, filename, false); + if (!deps) { + psError(PS_ERR_UNKNOWN, false, "failed to parse file: %s", filename); + return false; + } + if (nFail) { + psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing file: %s", nFail, filename); + psFree(deps); + return false; + } + + psMetadataItem *item = NULL; + psMetadataIterator *iter = psMetadataIteratorAlloc(deps, 0, NULL); + while ((item = psMetadataGetAndIncrement(iter))) { + if (item->type != PS_DATA_STRING) { + psError(PS_ERR_UNKNOWN, false, "file: %s is in the wrong format", filename); + psFree(iter); + psFree(deps); + return false; + } + + char *name = item->name; + char *dependsOn = item->data.str; + + if (!magicTreeInsert( + config->dbh, + magic_id, + name, + dependsOn + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(iter); + psFree(deps); + return false; + } + } + psFree(iter); + psFree(deps); + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/magictool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/magictool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/magictool.h (revision 22235) @@ -0,0 +1,47 @@ +/* + * magictool.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MAGICTOOL_H +#define MAGICTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + MAGICTOOL_MODE_NONE = 0x0, + MAGICTOOL_MODE_DEFINEBYQUERY, + MAGICTOOL_MODE_DEFINERUN, + MAGICTOOL_MODE_UPDATERUN, + MAGICTOOL_MODE_ADDINPUTSKYFILE, + MAGICTOOL_MODE_INPUTSKYFILE, + MAGICTOOL_MODE_TOTREE, + MAGICTOOL_MODE_INPUTTREE, + MAGICTOOL_MODE_REVERTTREE, + MAGICTOOL_MODE_TOPROCESS, + MAGICTOOL_MODE_INPUTS, + MAGICTOOL_MODE_ADDRESULT, + MAGICTOOL_MODE_REVERTNODE, + MAGICTOOL_MODE_TOMASK, + MAGICTOOL_MODE_ADDMASK, + MAGICTOOL_MODE_REVERTMASK, + MAGICTOOL_MODE_MASK, +} MAGICtoolMode; + +pxConfig *magictoolConfig(pxConfig *config, int argc, char **argv); + +#endif // MAGICTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/magictoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/magictoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/magictoolConfig.c (revision 22235) @@ -0,0 +1,197 @@ +/* + * magictoolConfig.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "magictool.h" + +pxConfig *magictoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id (required)", 0); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + + // -addinputskyfile + psMetadata *addinputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define difftool ID (required)", 0); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL); + + // -inputskyfile + psMetadata *inputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by difftool ID", 0); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-node", 0, "search by symbolic node name", NULL); + psMetadataAddU64(inputskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -totree + psMetadata *totreeArgs = psMetadataAlloc(); + psMetadataAddS64(totreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddU64(totreeArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(totreeArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -inputtree + psMetadata *inputtreeArgs = psMetadataAlloc(); + psMetadataAddS64(inputtreeArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); + psMetadataAddStr(inputtreeArgs, PS_LIST_TAIL, "-dep_file", 0, "order of operations dep. file", NULL); + psMetadataAddS16(inputtreeArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -reverttree + psMetadata *reverttreeArgs = psMetadataAlloc(); + psMetadataAddS64(reverttreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); + psMetadataAddS16(reverttreeArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -inputs + psMetadata *inputsArgs = psMetadataAlloc(); + psMetadataAddS64(inputsArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); + psMetadataAddStr(inputsArgs, PS_LIST_TAIL, "-node", 0, "search by symbolic node name", NULL); + psMetadataAddU64(inputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tooprocess + psMetadata *toprocessArgs = psMetadataAlloc(); + psMetadataAddS64(toprocessArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddU64(toprocessArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toprocessArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresult + psMetadata *addresultArgs = psMetadataAlloc(); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -revertnode + psMetadata *revertnodeArgs = psMetadataAlloc(); + psMetadataAddS64(revertnodeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); + psMetadataAddStr(revertnodeArgs, PS_LIST_TAIL, "-node", 0, "search by node name", NULL); + psMetadataAddS16(revertnodeArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -tomask + psMetadata *tomaskArgs = psMetadataAlloc(); + psMetadataAddU64(tomaskArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tomaskArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addmask + psMetadata *addmaskArgs = psMetadataAlloc(); + psMetadataAddS64(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); + psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddS32(addmaskArgs, PS_LIST_TAIL, "-streaks", 0, "define number of streaks", 0); + psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -revertmask + psMetadata *revertmaskArgs = psMetadataAlloc(); + psMetadataAddS64(revertmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); + psMetadataAddS16(revertmaskArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -mask + psMetadata *maskArgs = psMetadataAlloc(); + psMetadataAddS64(maskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0); + psMetadataAddU64(maskArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(maskArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "", MAGICTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-updaterun", "", MAGICTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-addinputskyfile", "", MAGICTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs); + PXOPT_ADD_MODE("-inputskyfile", "", MAGICTOOL_MODE_INPUTSKYFILE, inputskyfileArgs); + PXOPT_ADD_MODE("-totree", "", MAGICTOOL_MODE_TOTREE, totreeArgs); + PXOPT_ADD_MODE("-inputtree", "", MAGICTOOL_MODE_INPUTTREE, inputtreeArgs); + PXOPT_ADD_MODE("-reverttree", "", MAGICTOOL_MODE_REVERTTREE, reverttreeArgs); + PXOPT_ADD_MODE("-toprocess", "", MAGICTOOL_MODE_TOPROCESS, toprocessArgs); + PXOPT_ADD_MODE("-inputs", "", MAGICTOOL_MODE_INPUTS, inputsArgs); + PXOPT_ADD_MODE("-addresult", "", MAGICTOOL_MODE_ADDRESULT, addresultArgs); + PXOPT_ADD_MODE("-revertnode", "", MAGICTOOL_MODE_REVERTNODE, revertnodeArgs); + PXOPT_ADD_MODE("-tomask", "", MAGICTOOL_MODE_TOMASK, tomaskArgs); + PXOPT_ADD_MODE("-addmask", "", MAGICTOOL_MODE_ADDMASK, addmaskArgs); + PXOPT_ADD_MODE("-revertmask", "", MAGICTOOL_MODE_REVERTMASK, revertmaskArgs); + PXOPT_ADD_MODE("-mask", "", MAGICTOOL_MODE_MASK, maskArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pstamptool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pstamptool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pstamptool.c (revision 22235) @@ -0,0 +1,829 @@ +/* + * pstamptool.c + * + * Copyright (C) 2008 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "pstamptool.h" + +static bool adddatastoreMode(pxConfig *config); +static bool datastoreMode(pxConfig *config); +static bool moddatastoreMode(pxConfig *config); +static bool addReqMode(pxConfig *config); +static bool completedReqMode(pxConfig *config); +static bool listReqMode(pxConfig *config); +static bool pendingReqMode(pxConfig *config); +static bool updateReqMode(pxConfig *config); +static bool revertReqMode(pxConfig *config); +static bool addJobMode(pxConfig *config); +static bool listJobMode(pxConfig *config); +static bool pendingJobMode(pxConfig *config); +static bool updateJobMode(pxConfig *config); +static bool addProjectMode(pxConfig *config); +static bool projectMode(pxConfig *config); +static bool modProjectMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pstamptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(PSTAMPTOOL_MODE_ADDDATASTORE, adddatastoreMode); + MODECASE(PSTAMPTOOL_MODE_DATASTORE, datastoreMode); + MODECASE(PSTAMPTOOL_MODE_MODDATASTORE, moddatastoreMode); + MODECASE(PSTAMPTOOL_MODE_ADDREQ, addReqMode); + MODECASE(PSTAMPTOOL_MODE_COMPLETEDREQ, completedReqMode); + MODECASE(PSTAMPTOOL_MODE_LISTREQ, listReqMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGREQ, pendingReqMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEREQ, updateReqMode); + MODECASE(PSTAMPTOOL_MODE_REVERTREQ, revertReqMode); + MODECASE(PSTAMPTOOL_MODE_ADDJOB, addJobMode); + MODECASE(PSTAMPTOOL_MODE_LISTJOB, listJobMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingJobMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updateJobMode); + MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addProjectMode); + MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modProjectMode); + MODECASE(PSTAMPTOOL_MODE_PROJECT, projectMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool adddatastoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_STR(outProduct, config->args, "-out_product", true, false); + PXOPT_LOOKUP_STR(lastFileset, config->args, "-last_fileset", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + if (!pstampDataStoreInsert(config->dbh, + 0, + state, + lastFileset, + outProduct, + uri + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool datastoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ds_id", "ds_id", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_datastore.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampDataStore", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool moddatastoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", true, false); + PXOPT_LOOKUP_STR(lastFileset, config->args, "-last_fileset", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + if (!state && !lastFileset) { + psError(PS_ERR_UNKNOWN, true, "at least one of -last_fileset or -state is required"); + return false; + } + + char *query = psStringCopy ("UPDATE pstampDataStore SET"); + bool needComma = false; + + if (lastFileset) { + psStringAppend(&query, " lastFileset = '%s'", lastFileset); + needComma = true; + } + + if (state) { + psStringAppend(&query, "%s state = '%s'", needComma ? "," : " ", state); + needComma = true; // be ready in case we add another field + } + + psStringAppend(&query, " WHERE ds_id = %" PRId64, ds_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} + +static bool addReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + // PXOPT_LOOKUP_STR(outFileset, config->args, "-out_fileset", true, false); + PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); + + char *query ="INSERT INTO pstampRequest" + " (state, uri, ds_id, fault)" + " VALUES( 'new', '%s', %" PRId64 ", 0 )"; + if (!p_psDBRunQuery(config->dbh, query, uri, ds_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have affected one row but %" PRIu64 " rows were modified", + affected); + return false; + } + + psS64 req_id = psDBLastInsertID(config->dbh); + + printf("%" PRId64 "\n", req_id); + + return true; +} + +static bool pendingReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_pendingreq.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampRequest"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampRequest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool listReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = NULL; + psStringAppend(&query, "SELECT * from pstampRequest WHERE req_id = %" PRId64, req_id); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampRequest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool completedReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = + "SELECT * FROM pstampRequest WHERE state = 'run' AND " + "(SELECT count(*) FROM pstampJob WHERE pstampJob.req_id = pstampRequest.req_id " + " AND pstampJob.state != 'stop') = 0" ; + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampRequest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updateReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(outProduct, config->args, "-outProduct", false, false); + PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_STR(reqType, config->args, "-reqType", false, false); + + psString query = NULL; + psStringAppend(&query, "UPDATE pstampRequest SET"); + + char c = ' '; + if (state) { + psStringAppend(&query, "%c state = '%s'", c, state); + c = ','; + } + if (outProduct) { + psStringAppend(&query, "%c outProduct = '%s'", c, outProduct); + c = ','; + } + if (fault) { + psStringAppend(&query, "%c fault = %s", c, fault); + c = ','; + } + if (uri) { + psStringAppend(&query, "%c uri = '%s'", c, uri); + c = ','; + } + if (name) { + psStringAppend(&query, "%c name = '%s'", c, name); + c = ','; + } + if (reqType) { + psStringAppend(&query, "%c reqType = '%s'", c, reqType); + c = ','; + } + if (!state && !outProduct && !fault && !uri && !name && !reqType) { + psError(PS_ERR_UNKNOWN, true, "at least one of state, outProduct, fault, uri, name, and reqType must be specified"); + return false; + } + + psStringAppend(&query, " WHERE req_id = %" PRId64, req_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + // note zero is not an error + if (affected > 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} + +static bool revertReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + + // printf("Revert request %" PRId64 "\n", req_id); + + if (!p_psDBRunQuery(config->dbh, "DELETE FROM pstampJob where req_id = %" PRId64, req_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!p_psDBRunQuery(config->dbh, + "UPDATE pstampRequest set state ='new', name=NULL, reqType=NULL, fault=0 where req_id = %" PRId64, req_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool addJobMode(pxConfig *config) +{ + bool stampJob = false; + char *query = NULL; + + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + PXOPT_LOOKUP_STR(rownum, config->args, "-rownum", true, false); + PXOPT_LOOKUP_STR(job_type, config->args, "-job_type", false, false); + PXOPT_LOOKUP_STR(outputBase, config->args, "-outputBase", true, false); + PXOPT_LOOKUP_STR(argString, config->args, "-args", false, false); + PXOPT_LOOKUP_STR(stateString, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + + // default value for job_type is defined in pstamptoolConfig.c + if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query")) { + stampJob = false; + } else if (!strcmp(job_type, "stamp")) { + stampJob = true; + } else { + psError(PS_ERR_UNKNOWN, false, "unknown value for -job_type: %s", job_type); + return false; + } + + if (stampJob && !argString) { + psError(PS_ERR_UNKNOWN, true, "-args is required for stamp job"); + return false; + } + + if (stampJob) { + query = pxDataGet("pstamptool_addjob_stampjob.sql"); + } else { + query = pxDataGet("pstamptool_addjob_otherjob.sql"); + } + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query, req_id, rownum, stateString, job_type, uri, exp_id, outputBase, fault, argString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have affected one row but %" PRIu64 " rows were modified", + affected); + return false; + } + + psS64 job_id = psDBLastInsertID(config->dbh); + printf("%" PRId64 "\n", job_id); + + return true; +} + +static bool listJobMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", false, false); + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", false, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = NULL; + + if (!req_id && !job_id) { + fprintf(stderr, "either req_id or job_id must be specified\n"); + exit (1); + } + + if (req_id) { + psStringAppend(&query, + "SELECT" + " *" + " FROM pstampJob" + " WHERE req_id = %" PRId64, req_id + ); + } else { + psStringAppend(&query, + "SELECT" + " *" + " FROM pstampJob" + " WHERE job_id = %" PRId64, job_id + ); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampJob", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingJobMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_pendingjob.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampJob", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updateJobMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); + + psString faultStr = NULL; + if (!fault) { + faultStr = psStringCopy(""); + } else { + psStringAppend(&faultStr, ", fault = '%s'", fault); + } + + char *query ="UPDATE pstampJob" + " SET state = '%s' %s" + " WHERE job_id = %" PRId64; + + if (!p_psDBRunQuery(config->dbh, query, state, faultStr, job_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(faultStr); + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} +static bool addProjectMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(imagedb, config->args, "-imagedb", true, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false); + + if (!pstampProjectInsert(config->dbh, + 0, + name, + state, + imagedb, + dvodb, + camera, + telescope, + need_magic + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool projectMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_project.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampDataStore", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool modProjectMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(proj_id, config->args, "-proj_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + char *query = psStringCopy ("UPDATE pstampProject SET"); + + psStringAppend(&query, " state = '%s'", state); + + psStringAppend(&query, " WHERE proj_id = %" PRId64, proj_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pstamptool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pstamptool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pstamptool.h (revision 22235) @@ -0,0 +1,48 @@ +/* + * pstamptool.h + * + * Copyright (C) 2008 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PSTAMPTOOL_H +#define PSTAMPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + PSTAMPTOOL_MODE_NONE = 0x0, + PSTAMPTOOL_MODE_ADDDATASTORE, + PSTAMPTOOL_MODE_DATASTORE, + PSTAMPTOOL_MODE_MODDATASTORE, + PSTAMPTOOL_MODE_ADDREQ, + PSTAMPTOOL_MODE_LISTREQ, + PSTAMPTOOL_MODE_COMPLETEDREQ, + PSTAMPTOOL_MODE_PENDINGREQ, + PSTAMPTOOL_MODE_UPDATEREQ, + PSTAMPTOOL_MODE_REVERTREQ, + PSTAMPTOOL_MODE_ADDJOB, + PSTAMPTOOL_MODE_LISTJOB, + PSTAMPTOOL_MODE_PENDINGJOB, + PSTAMPTOOL_MODE_JOBRESULT, + PSTAMPTOOL_MODE_UPDATEJOB, + PSTAMPTOOL_MODE_ADDPROJECT, + PSTAMPTOOL_MODE_MODPROJECT, + PSTAMPTOOL_MODE_PROJECT, +} pstamptoolMode; + +pxConfig *pstamptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // PSTAMPTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pstamptoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pstamptoolConfig.c (revision 22235) @@ -0,0 +1,203 @@ +/* + * pstamptoolConfig.c + * + * Copyright (C) 2008 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pstamptool.h" + +pxConfig *pstamptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + // -adddatastore + psMetadata *adddatastoreArgs = psMetadataAlloc(); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-uri", 0, "define storage uri (required)", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-out_product", 0, "define output product name (required)", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-last_fileset", 0, "define last fileset seen", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-state", 0, "define datastore state", "enabled"); + + // -datastore + psMetadata *datastoreArgs = psMetadataAlloc(); + psMetadataAddS64(datastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddBool(datastoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -moddatastore + psMetadata *moddatastoreArgs = psMetadataAlloc(); + psMetadataAddS64(moddatastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-last_fileset", 0, "set last_fileset seen", NULL); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); + + // -addreq + psMetadata *addreqArgs = psMetadataAlloc(); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); + psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); + // psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-out_fileset", 0, "define request output_fileset", NULL); + + // -pendingreq + psMetadata *pendingreqArgs = psMetadataAlloc(); + psMetadataAddS64(pendingreqArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id", 0); + psMetadataAddU64(pendingreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -listreq + psMetadata *listreqArgs = psMetadataAlloc(); + psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id (required)", 0); + psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -completedreq + psMetadata *completedreqArgs = psMetadataAlloc(); + psMetadataAddU64(completedreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(completedreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updatereq + psMetadata *updatereqArgs = psMetadataAlloc(); + psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to change state", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-outProduct", 0, "define request outProduct", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "define request fault code", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-uri", 0, "define the uri", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "define the name", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "define the reqType", NULL); + + // -revertreq + psMetadata *revertreqArgs = psMetadataAlloc(); + psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to revert", 0); + + // -addjob + psMetadata *addjobArgs = psMetadataAlloc(); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-uri", 0, "define job file uri", NULL); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-args", 0, "define job args", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "new state", "run"); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "exposure id", 0); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + + // -listjob + psMetadata *listjobArgs = psMetadataAlloc(); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); + psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingjob + psMetadata *pendingjobArgs = psMetadataAlloc(); + psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); + psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); + psMetadataAddU64(pendingjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updatejob + psMetadata *updatejobArgs = psMetadataAlloc(); + psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); + psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); + // psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-result", 0, "new result", NULL); + psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + + // -addproject + psMetadata *addprojectArgs = psMetadataAlloc(); + psMetadataAddStr(addprojectArgs, PS_LIST_TAIL, "-name", 0, "define project name (required)", NULL); + psMetadataAddStr(addprojectArgs, PS_LIST_TAIL, "-state", 0, "define state for project (enabled, disabled)", "enabled"); + psMetadataAddStr(addprojectArgs, PS_LIST_TAIL, "-imagedb", 0, "define name of database for project (required)", NULL); + psMetadataAddStr(addprojectArgs, PS_LIST_TAIL, "-dvodb", 0, "define name of dvo database for project", NULL); + psMetadataAddStr(addprojectArgs, PS_LIST_TAIL, "-inst", 0, "define name of camera for project (required)", NULL); + psMetadataAddStr(addprojectArgs, PS_LIST_TAIL, "-telescope", 0, "define name of telescope for project (required)", NULL); + psMetadataAddBool(addprojectArgs, PS_LIST_TAIL, "-need_magic", 0, "define need_magic for project", false); + + // -modproject + psMetadata *modprojectArgs = psMetadataAlloc(); + psMetadataAddS64(modprojectArgs, PS_LIST_TAIL, "-proj_id", 0, "define project ID to modify (required)", 0); + psMetadataAddStr(modprojectArgs, PS_LIST_TAIL, "-imagedb", 0, "define name of database for project", NULL); + psMetadataAddStr(modprojectArgs, PS_LIST_TAIL, "-state", 0, "define state for project (enabled, disabled)", NULL); + psMetadataAddStr(modprojectArgs, PS_LIST_TAIL, "-dvodb", 0, "define name of dvo database for project", NULL); + psMetadataAddStr(modprojectArgs, PS_LIST_TAIL, "-camera", 0, "define name of camera for project", NULL); + psMetadataAddStr(modprojectArgs, PS_LIST_TAIL, "-telescope", 0, "define name of telescope for project", NULL); + psMetadataAddBool(modprojectArgs, PS_LIST_TAIL, "-need_magic", 0, "define need_magic for project", false); + + // -project + psMetadata *projectArgs = psMetadataAlloc(); + psMetadataAddStr(projectArgs, PS_LIST_TAIL, "-name", 0, "define project name to list (required)", NULL); + psMetadataAddBool(projectArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-addreq", "", PSTAMPTOOL_MODE_ADDREQ, addreqArgs); + PXOPT_ADD_MODE("-pendingreq", "", PSTAMPTOOL_MODE_PENDINGREQ, pendingreqArgs); + PXOPT_ADD_MODE("-updatereq", "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs); + PXOPT_ADD_MODE("-listreq", "", PSTAMPTOOL_MODE_LISTREQ, listreqArgs); + PXOPT_ADD_MODE("-completedreq", "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs); + PXOPT_ADD_MODE("-revertreq", "", PSTAMPTOOL_MODE_REVERTREQ, revertreqArgs); + + PXOPT_ADD_MODE("-addjob", "", PSTAMPTOOL_MODE_ADDJOB, addjobArgs); + PXOPT_ADD_MODE("-listjob", "", PSTAMPTOOL_MODE_LISTJOB, listjobArgs); + PXOPT_ADD_MODE("-pendingjob", "", PSTAMPTOOL_MODE_PENDINGJOB, pendingjobArgs); + PXOPT_ADD_MODE("-updatejob", "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs); + + PXOPT_ADD_MODE("-adddatastore", "", PSTAMPTOOL_MODE_ADDDATASTORE, adddatastoreArgs); + PXOPT_ADD_MODE("-datastore", "", PSTAMPTOOL_MODE_DATASTORE, datastoreArgs); + PXOPT_ADD_MODE("-moddatastore", "", PSTAMPTOOL_MODE_MODDATASTORE, moddatastoreArgs); + + PXOPT_ADD_MODE("-addproject", "", PSTAMPTOOL_MODE_ADDPROJECT, addprojectArgs); + PXOPT_ADD_MODE("-modproject", "", PSTAMPTOOL_MODE_MODPROJECT, modprojectArgs); + PXOPT_ADD_MODE("-project", "", PSTAMPTOOL_MODE_PROJECT, projectArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxadmin.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxadmin.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxadmin.c (revision 22235) @@ -0,0 +1,221 @@ +/* + * pxadmin.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pxtools.h" +#include "pxadmin.h" + +bool createMode(pxConfig *config); +bool deleteMode(pxConfig *config); +static bool runMultipleStatments(pxConfig *config, const char *query); + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pxAdminConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + case PXADMIN_MODE_RECREATE: + if (!deleteMode(config)) { + goto FAIL; + } + // fall through + case PXADMIN_MODE_CREATE: + if (!createMode(config)) { + goto FAIL; + } + break; + case PXADMIN_MODE_DELETE: + if (!deleteMode(config)) { + goto FAIL; + } + break; + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +bool createMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("pxadmin_create_tables.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // BEGIN + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!runMultipleStatments(config, query)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // COMMIT + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +bool deleteMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + { + char line[128], answer[128]; + + psMetadataItem *name = pmConfigUserSite(config->modules, "DBNAME", PS_DATA_STRING); + if (!name) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine database name."); + return false; + } + psString dbName = name->data.str; + + fprintf(stdout, "*** delete the tables from database %s? ***\n", dbName); + fprintf(stdout, "*** to delete the tables, answer YES, and give password ***\n"); + fprintf(stdout, "*** WARNING: this action is permanent ***\n\n"); + + fprintf(stdout, "delete the tables (YES/[n]): "); + if (!fgets(line, 128, stdin)) { + psError(PS_ERR_IO, true, "Unable to read response."); + return false; + } + sscanf(line, "%s", answer); + if (strcmp (answer, "YES")) { + psError(PS_ERR_UNKNOWN, true, "tables NOT deleleted"); + return false; + } + + fprintf(stdout, "enter dbh connection password: "); + if (!fgets(line, 128, stdin)) { + psError(PS_ERR_IO, true, "Unable to read response."); + return false; + } + sscanf(line, "%s", answer); + + psMetadataItem *pass = pmConfigUserSite(config->modules, "DBPASSWORD", PS_DATA_STRING); + if (!pass) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine database password."); + return false; + } + psString dbPassword = pass->data.str; + if (strcmp (answer, dbPassword)) { + psError(PS_ERR_UNKNOWN, true, "invalid passwd - tables NOT deleleted"); + return false; + } + } + + psString query = pxDataGet("pxadmin_drop_tables.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // BEGIN + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!runMultipleStatments(config, query)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // COMMIT + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool runMultipleStatments(pxConfig *config, const char *query) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(query, false); + + // loop over all statements in string + psList *statements = psStringSplit(query, ";", false); + + psString q = NULL; + psListIterator *iter = psListIteratorAlloc(statements, PS_LIST_HEAD, false); + while ((q = psListGetAndIncrement(iter))) { + if (!p_psDBRunQuery(config->dbh, q)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(iter); + psFree(statements); + return false; + } + } + psFree(iter); + psFree(statements); + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxadmin.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxadmin.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxadmin.h (revision 22235) @@ -0,0 +1,34 @@ +/* + * pxadmin.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXADMIN_H +#define PXADMIN_H 1 + +#include "pxtools.h" + +typedef enum { + PXADMIN_MODE_NONE = 0x0, + PXADMIN_MODE_CREATE, + PXADMIN_MODE_DELETE, + PXADMIN_MODE_RECREATE +} pxAdminMode; + +pxConfig *pxAdminConfig (pxConfig *config, int argc, char **argv); + +#endif // PXADMIN_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxadminConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxadminConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxadminConfig.c (revision 22235) @@ -0,0 +1,153 @@ +/* + * pxadminConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxadmin.h" + +static void pxAdminShowDB (const pxConfig *config, const char *program) +{ + fprintf (stderr, "\nPan-STARRS DataBase Admin Tool\n\n"); + fprintf (stderr, "Usage: %s [mode]\n", program); + fprintf (stderr, " [mode] : -create | -delete\n\n"); + + psMetadataItem *server = pmConfigUserSite(config->modules, "DBSERVER", PS_DATA_STRING); + psMetadataItem *user = pmConfigUserSite(config->modules, "DBUSER", PS_DATA_STRING); + psMetadataItem *name = pmConfigUserSite(config->modules, "DBNAME", PS_DATA_STRING); + psMetadataItem *port = pmConfigUserSite(config->modules, "DBPORT", PS_TYPE_S32); + + if (!server || !user || !name) { + psErrorClear(); + psWarning("Unable to determine database connection details."); + return; + } + if (!port) { + psErrorClear(); + } + + fprintf (stderr, "connecting to %s as %s (port %d)\n", server->data.str, user->data.str, + port ? port->data.S32 : 0); + fprintf (stderr, "using database %s\n\n", name->data.str); + + return; +} + +pxConfig *pxAdminConfig(pxConfig *config, int argc, char **argv) +{ + if (!psArgumentGet(argc, argv, "-dbname")) { + psError(PS_ERR_UNEXPECTED_NULL, true, + "Command-line argument '-dbname XXX' is required, for database safety"); + return NULL; + } + + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // Parse other command-line arguments + psMetadata *arguments = psMetadataAlloc(); // The arguments, with default values + + int N; + config->mode = PXADMIN_MODE_NONE; + if ((N = psArgumentGet(argc, argv, "-create"))) { + psArgumentRemove(N, &argc, argv); + if (config->mode) { + psAbort("only one mode selection is allowed"); + } + config->mode = PXADMIN_MODE_CREATE; + } + if ((N = psArgumentGet(argc, argv, "-delete"))) { + psArgumentRemove(N, &argc, argv); + if (config->mode) { + psAbort("only one mode selection is allowed"); + } + config->mode = PXADMIN_MODE_DELETE; + } + if ((N = psArgumentGet(argc, argv, "-recreate"))) { + psArgumentRemove(N, &argc, argv); + if (config->mode) { + psAbort("only one mode selection is allowed"); + } + config->mode = PXADMIN_MODE_RECREATE; + } + + // paul's argument parsing convention requires: -key value + psMetadataAddStr(arguments, PS_LIST_TAIL, "-create", 0, + "create all IPP tables", ""); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-delete", 0, + "delete all IPP tables", ""); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-recreate", 0, + "delete and recreate all tables", ""); + + if (config->mode == PXADMIN_MODE_NONE) { + pxAdminShowDB (config, argv[0]); + fprintf (stderr, "admin mode not specified\n"); + + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + return NULL; + } + + if ((N = psArgumentGet (argc, argv, "-help"))) { + pxAdminShowDB (config, argv[0]); + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + return NULL; + } + + if (! psArgumentParse(arguments, &argc, argv) || argc != 1) { + pxAdminShowDB (config, argv[0]); + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + return NULL; + } + + psFree(arguments); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + // save argv/argc + config->argv = argv; + config->argc = argc; + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxcam.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxcam.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxcam.c (revision 22235) @@ -0,0 +1,268 @@ +/* + * pxcam.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxcam.h" + +bool pxcamSetSearchArgs (psMetadata *md) { + + psMetadataAddS64(md, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(md, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(md, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + + return true; +} + +bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where) { + + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + return true; +} + +bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid camRun state: %s", state); + return false; + } + + char *query = "UPDATE camRun SET state = '%s' WHERE cam_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, cam_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for cam_id %" PRId64, cam_id); + return false; + } + + return true; +} + + +bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid chipRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET camRun.state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE camRun SET camRun.label = '%s' WHERE cam_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, label, cam_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for cam_id %" PRId64, cam_id); + return false; + } + + return true; +} + +bool pxcamRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET camRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxcamQueueByChipID(pxConfig *config, + psS64 chip_id, + char *workdir, + char *label, + char *recipe, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("camtool_queue_chip_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQuery(config->dbh, query, + "new", // state + workdir ? workdir : "NULL", + "dirty", //workdir_state + label ? label : "NULL", + recipe ? recipe : "NULL", + expgroup ? expgroup : "NULL", + dvodb ? dvodb : "NULL", + tess_id ? tess_id : "NULL", + end_stage ? end_stage : "NULL", + (long long)chip_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + return false; + } + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxcam.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxcam.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxcam.h (revision 22235) @@ -0,0 +1,45 @@ +/* + * pxcam.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXCAM_H +#define PXCAM_H 1 + +#include + +#include "pxtools.h" + +bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state); +bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label); +bool pxcamRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +bool pxcamSetSearchArgs (psMetadata *md); +bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where); + +bool pxcamQueueByChipID(pxConfig *config, + psS64 chip_id, + char *workdir, + char *label, + char *recipe, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage); + +#endif // PXCAM_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxchip.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxchip.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxchip.c (revision 22235) @@ -0,0 +1,255 @@ +/* + * pxchip.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxchip.h" + +bool pxchipSetSearchArgs (psMetadata *md) { + + // XXX need to allow multiple exp_ids + psMetadataAddS64(md, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + // psMetadataAddStr(md, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + // psMetadataAddStr(md, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(md, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "search by min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "search by min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "search by max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "search by min", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "search by max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "search by min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "search by min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(md, PS_LIST_TAIL, "-solang_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-solang_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + return true; +} + +// XXX explicit refs to rawExp and chipRun? +// XXX careful with entries that could match rawExp rawImfile, chipRun, or chipProcessedImfile (eg, bg) +bool pxchipGetSearchArgs (pxConfig *config, psMetadata *where) { + + // definebyquery : rawExp only + // updaterun : rawExp, chipRun + // pendingimfile : rawExp, chipRun + // processedimfile : rawExp, chipRun, chipProcessedImfile + // revertprocessedimfile : rawExp, chipProcessedImfile + // updateprocessedimfile : chipProcessedImfile + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + return true; +} + +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid chipRun state: %s", state); + return false; + } + + char *query = "UPDATE chipRun SET state = '%s' WHERE chip_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, chip_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for chip_id %" PRId64, chip_id); + return false; + } + + return true; +} + + +bool pxchipRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid chipRun state: %s", state); + return false; + } + + + psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxchipRunSetLabel(pxConfig *config, psS64 chip_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE chipRun SET label = '%s' WHERE chip_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, label, chip_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for chip_id %" PRId64, chip_id); + return false; + } + + return true; +} + + +bool pxchipRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +psS64 pxchipQueueByExpTag(pxConfig *config, + psS64 exp_id, + const char *workdir, + const char *label, + const char *reduction, + const char *expgroup, + const char *dvodb, + const char *tess_id, + const char *end_stage) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // create a chipRun + if (!chipRunInsert(config->dbh, + 0x0, // chip_id + exp_id, + "new", // state + workdir, + "dirty", // workdir_state + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return 0; + } + + return psDBLastInsertID(config->dbh); +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxchip.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxchip.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxchip.h (revision 22235) @@ -0,0 +1,46 @@ +/* + * pxchip.h + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXCHIP_H +#define PXCHIP_H 1 + +#include + +#include "pxtools.h" + +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state); +bool pxchipRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxchipRunSetLabel(pxConfig *config, psS64 chip_id, const char *label); +bool pxchipRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +psS64 pxchipQueueByExpTag(pxConfig *config, + psS64 exp_id, + const char *workdir, + const char *label, + const char *reduction, + const char *expgroup, + const char *dvodb, + const char *tess_id, + const char *end_stage); + + +bool pxchipSetSearchArgs (psMetadata *md); +bool pxchipGetSearchArgs (pxConfig *config, psMetadata *where); + +#endif // PXCHIP_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxconfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxconfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxconfig.c (revision 22235) @@ -0,0 +1,227 @@ +/* + * pxconfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include +#include + +#include "pxtools.h" + +static void pxConfigFree(pxConfig *ptr); + +pxConfig *pxConfigAlloc(void) +{ + pxConfig *config; + + config = psAlloc(sizeof(pxConfig)); + psMemSetDeallocator(config, (psFreeFunc)pxConfigFree); + + config->modeName = NULL; + config->mode = 0; + config->dbh = NULL; + config->modules = NULL; + // XXX config->where = NULL; + config->args = NULL; + + return config; +} + +static void pxConfigFree(pxConfig *config) +{ + psFree(config->modeName); + psFree(config->dbh); + psFree(config->modules); + // XXX psFree(config->where); + psFree(config->args); +} + +void pxUsage(FILE *stream, int argc, char **argv, const char *modeName, psMetadata *argSet) +{ + fprintf(stream, "Usage: %s %s []\n\n", argv[0], modeName); + fprintf(stream, "%s:\n", modeName); + + psArgumentHelpSimple(stream, argSet); +} + +bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets) +{ + // figure out what mode we're running in + psMetadataIterator *iter = psMetadataIteratorAlloc(modes, PS_LIST_HEAD, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement(iter))) { + int N = 0; + if ((N = psArgumentGet(argc, argv, item->name))) { + psArgumentRemove(N, &argc, argv); + // check for duplicate mode specification + if (config->mode) { + psError(PS_ERR_UNKNOWN, true, "only one mode selection is allowed"); + fprintf(stream, "only one mode selection is allowed\n"); + pxUsage(stream, argc, argv, "", modes); + psFree(iter); + return NULL; + } + + config->mode = item->data.U32; + config->modeName = psStringCopy(item->name); + + bool status = false; + psMetadata *argset = psMetadataLookupMetadata(&status, argSets, item->name); + // make sure we can find the argSet for the sepcified mode + if (!status) { + psError(PS_ERR_UNKNOWN, true, "can not find arguments for mode"); + fprintf(stderr, "can not find arugments for mode"); + psFree(iter); + return NULL; + } + + config->args = psMemIncrRefCounter(argset); + } + + } + + psFree(iter); + + // make sure we found a mode + if (config->mode == PXTOOL_MODE_NONE) { + psError(PS_ERR_UNKNOWN, true, "mode argument is required"); + fprintf(stderr, "mode argument is required\n"); + pxUsage(stream, argc, argv, "", modes); + return NULL; + } + + // actually parse the command line + if (!psArgumentParse(config->args, &argc, argv)) { + psError(PS_ERR_UNKNOWN, false, "error parsing arguments"); + fprintf(stderr, "error parsing arguments\n"); + pxUsage(stream, argc, argv, config->modeName, config->args); + return NULL; + } + + // look for left overs on the command line + if (argc != 1) { + psError(PS_ERR_UNKNOWN, false, "extra arguments: "); + fprintf(stderr, "extra arguments: "); + for (int i = 1; i < argc; i++) { + fprintf (stderr, "%s ", argv[i]); + } + fprintf(stderr, "\n"); + pxUsage(stream, argc, argv, config->modeName, config->args); + return NULL; + } + + // make sure that all required parameters have been specified + iter = psMetadataIteratorAlloc(config->args, PS_LIST_HEAD, NULL); + item = NULL; // Item from iterator + while ((item = psMetadataGetAndIncrement(iter))) { + if (strstr(item->comment, "require") == NULL) { + continue; + } + if (strstr(item->comment, "(found)") != NULL) { + continue; + } + + switch (item->type) { + case PS_DATA_BOOL: + psError(PS_ERR_UNKNOWN, false, "boolean type can not be required"); + fprintf(stderr, "boolean type can not be required\n"); + psFree(iter); + return NULL; + break; + case PS_DATA_S8: + if (item->data.S8 != INT8_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_S16: + if (item->data.S16 != INT16_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_S32: + if (item->data.S32 != INT32_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_S64: + if (item->data.S64 != INT64_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U8: + if (item->data.U8 != UINT8_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U16: + if (item->data.U16 != UINT16_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U32: + if (item->data.U32 != UINT32_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U64: + if (item->data.U64 != UINT64_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_F32: + case PS_DATA_F64: + if (!isnan(item->data.F64)) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_STRING: + case PS_DATA_TIME: + case PS_DATA_METADATA: + if (item->data.V) { + break; + } + goto ARG_REQUIRED; + default: + psError(PS_ERR_UNKNOWN, false, "unknown argument type"); + fprintf(stream, "unknown argument type\n"); + psFree(iter); + return NULL; + ARG_REQUIRED: + psError(PS_ERR_UNKNOWN, false, "argument %s is required", item->name); + fprintf(stream, "argument %s is required\n", item->name); + pxUsage(stream, argc, argv, config->modeName, config->args); + psFree(iter); + return NULL; + } + + } + + psFree(iter); + + // save argv/argc + config->argv = argv; + config->argc = argc; + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxconfig.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxconfig.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxconfig.h (revision 22235) @@ -0,0 +1,39 @@ +/* + * pxconfig.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXCONFIG_H +#define PXCONFIG_H 1 + +#include "pslib.h" +#include "psmodules.h" + +typedef struct { + char *modeName; + int mode; + pmConfig *modules; + psDB *dbh; + psMetadata *args; + // XXX deprecate psMetadata *where; + int argc; + char **argv; +} pxConfig; + +pxConfig *pxConfigAlloc(void); + +#endif // PXCONFIG_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxdata.c.template =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxdata.c.template (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxdata.c.template (revision 22235) @@ -0,0 +1,145 @@ +/* + * pxdata.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include // getenv +#include +#include +#include + +#include + +#include "pxtools.h" +#include "pxdata.h" + +static psHash *cachedData = NULL; + +static int test_f(const char *path); + +psString pxDataGet(const char *filename) +{ + PS_ASSERT_PTR_NON_NULL(filename, NULL); + + { + psString filepath = NULL; + + // look to see if PXDATA is set + char *PXDATA = getenv("PXDATA"); + if (PXDATA) { + // if it is, then prepend filename to PXDATA + psStringAppend(&filepath, "%s/%s", PXDATA, filename); + } + + // see if we have a valid filename + if (test_f(filepath)) { + // if we do, slurp the file + psString data = psSlurpFilename(filepath); + if (!data) { + psError(PS_ERR_IO, false, "failed to slurp %s", filepath); + psFree(filepath); + return NULL; + } + psTrace("pxdata", PS_LOG_INFO, "slupred %s", filepath); + psFree(filepath); + + return data; + } + + psFree(filepath); + } + + { + psString filepath = NULL; + + // look under our share path + psStringAppend(&filepath, "%s/%s", "@PKGDATADIR@", filename); + + // see if we have a valid filename + if (test_f(filepath)) { + // if we do, slurp the file + psString data = psSlurpFilename(filepath); + if (!data) { + psError(PS_ERR_IO, false, "failed to slurp %s", filepath); + psFree(filepath); + return NULL; + } + psTrace("pxdata", PS_LOG_INFO, "slupred %s", filepath); + psFree(filepath); + + return data; + } + + psFree(filepath); + } + + // couldn't find a matching filename + psError(PS_ERR_IO, true, "can't find a file in search path(s) to match : %s", filename); + + return NULL; +} + +static int test_f(const char *path) +{ + // copied from coreutils 6.4 test.c licenced under GPL v2 + struct stat stat_buf; + return (stat (path, &stat_buf) == 0 + && S_ISREG (stat_buf.st_mode)); +} + +psString pxDataGetCached(const char *filename) +{ + bool persistence = p_psMemAllocatePersistent(true); + + // make sure the cache is initalized + if (!cachedData) { + cachedData = psHashAlloc(10); + } + + // look to see if we've cached this file + { + psString data = psHashLookup(cachedData, filename); + if (data) { + p_psMemAllocatePersistent(persistence); + return psStringCopy(data); + } + } + + // we didn't have a cached copy so we need to try to read the file + psString data = pxDataGet(filename); + if (!data) { + p_psMemAllocatePersistent(persistence); + psError(PS_ERR_IO, false, "pxDataGet(%s) failed", filename); + return NULL; + } + + // store a copy of the file + if (!psHashAdd(cachedData, filename, data)) { + psFree(data); + p_psMemAllocatePersistent(persistence); + psError(PS_ERR_UNKNOWN, false, "psHashAdd() failed"); + return NULL; + } + + p_psMemAllocatePersistent(persistence); + + return psStringCopy(data); +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxdata.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxdata.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxdata.h (revision 22235) @@ -0,0 +1,28 @@ +/* + * pxdata.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXDATA_H +#define PXDATA_H 1 + +#include + +psString pxDataGet(const char *filename); +psString pxDataGetCached(const char *filename); + +#endif // PXDATA_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxerrors.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxerrors.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxerrors.c (revision 22235) @@ -0,0 +1,42 @@ +/* + * pxerrors.c + * + * Copyright (C) 2006 Eugene Magnier + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "pxtools.h" + +psExit pxerrorGetExitStatus () { + + psErrorCode err = psErrorCodeLast (); + switch (err) { + case PXTOOLS_ERR_SYS: + return PS_EXIT_SYS_ERROR; + case PXTOOLS_ERR_CONFIG: + return PS_EXIT_CONFIG_ERROR; + case PXTOOLS_ERR_PROG: + return PS_EXIT_PROG_ERROR; + case PXTOOLS_ERR_DATA: + return PS_EXIT_DATA_ERROR; + default: + return PS_EXIT_UNKNOWN_ERROR; + } + return PS_EXIT_UNKNOWN_ERROR; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxfake.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxfake.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxfake.c (revision 22235) @@ -0,0 +1,178 @@ +/* + * pxfake.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxcam.h" + +bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid fakeRun state: %s", state); + return false; + } + + char *query = "UPDATE fakeRun SET state = '%s' WHERE fake_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, fake_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for fake_id %" PRId64, fake_id); + return false; + } + + return true; +} + + +bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid fakeRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE fakeRun SET fakeRun.label = '%s' WHERE fake_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, label, fake_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for fake_id %" PRId64, fake_id); + return false; + } + + return true; +} + + +bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET fakeRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +psS64 pxfakeQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *reduction, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psTrace("pxtool", PS_LOG_INFO, "attemping to queue cam_id: %"PRId64, cam_id); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("faketool_queue_cam_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQuery(config->dbh, query, + "new", // state + workdir ? workdir : "NULL", + label ? label : "NULL", + reduction? reduction: "NULL", + expgroup ? expgroup : "NULL", + dvodb ? dvodb : "NULL", + tess_id ? tess_id : "NULL", + end_stage ? end_stage : "NULL", + (long long)cam_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have changed just one row"); + return false; + } + + return psDBLastInsertID(config->dbh); +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxfake.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxfake.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxfake.h (revision 22235) @@ -0,0 +1,44 @@ +/* + * pxfake.h + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXFAKE_H +#define PXFAKE_H 1 + +#include + +#include "pxtools.h" + +bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state); +bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label); +bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + + +psS64 pxfakeQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *reduction, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage); + + +#endif // PXFAKE_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxfault.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxfault.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxfault.c (revision 22235) @@ -0,0 +1,65 @@ +/* + * pxfault.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" + +bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code) +{ + PS_ASSERT_PTR_NON_NULL(dbh, false); + PS_ASSERT_PTR_NON_NULL(tableName, false); + PS_ASSERT_PTR_NON_NULL(where, false); + +#if 0 + // map code string to numeric fault code + psU32 code = mapCodeStrToInt(codeStr); + if (code == (psU32)-1) { + psError(PS_ERR_UNKNOWN, false, "error resolving error code"); + return false; + } +#endif + + // update the database + psMetadata *values = psMetadataAlloc(); + if (!psMetadataAddS16(values, PS_LIST_HEAD, "fault", 0, NULL, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to add metadata item fault"); + psFree(values); + return false; + } + + long rowsAffected = psDBUpdateRows(dbh, tableName, where, values); + psFree(values); + if (rowsAffected < 0) { + // database error + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (rowsAffected < 1) { + // we didn't do anything + psError(PS_ERR_UNKNOWN, false, "zero rows were affected - either the search criteria didn't match any rows or the field already had the value being set."); + return false; + } + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxinject.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxinject.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxinject.c (revision 22235) @@ -0,0 +1,163 @@ +/* + * pxinject.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxinject.h" + +static bool newExpMode(pxConfig *config); +static bool newImfileMode(pxConfig *config); +static bool updatenewExpMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pxinjectConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(PXINJECT_MODE_NEWEXP, newExpMode); + MODECASE(PXINJECT_MODE_NEWIMFILE, newImfileMode); + MODECASE(PXINJECT_MODE_UPDATENEWEXP, updatenewExpMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint (stderr, "failure\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool newExpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(tmp_exp_name, config->args, "-tmp_exp_name", true, false); + PXOPT_LOOKUP_STR(tmp_camera, config->args, "-tmp_inst", true, false); + PXOPT_LOOKUP_STR(tmp_telescope, config->args, "-tmp_telescope", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!newExpInsert(config->dbh, + 0, // exp_id + tmp_exp_name, + tmp_camera, + tmp_telescope, + "reg", // state + workdir, + "dirty", + reduction, + dvodb, + tess_id, + end_stage, + label, + NULL // epoch + ) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psS64 exp_id = psDBLastInsertID(config->dbh); + + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddS64(md, PS_LIST_TAIL, "exp_id", 0, NULL, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(md); + } + + // negate simple so the default is true + if (!ippdbPrintMetadata(stdout, md, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(md); + return false; + } + + psFree(md); + + return true; +} + +static bool newImfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(tmp_class_id, config->args, "-tmp_class_id", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + // insert with error flag state set to 0 (no errors) + if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updatenewExpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (state) { + // set detRun.state to state + return pxnewExpSetState(config, exp_id, state); + } + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxinject.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxinject.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxinject.h (revision 22235) @@ -0,0 +1,34 @@ +/* + * pxinject.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXINJECT_H +#define PXINJECT_H 1 + +#include "pxtools.h" + +typedef enum { + PXINJECT_MODE_NONE = 0x0, + PXINJECT_MODE_NEWEXP, + PXINJECT_MODE_NEWIMFILE, + PXINJECT_MODE_UPDATENEWEXP +} pxinjectMode; + +pxConfig *pxinjectConfig(pxConfig *config, int argc, char **argv); + +#endif // PXINJECT_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxinjectConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxinjectConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxinjectConfig.c (revision 22235) @@ -0,0 +1,98 @@ +/* + * pxinjectConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxinject.h" + +pxConfig *pxinjectConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -newExp + psMetadata *newExpArgs = psMetadataAlloc(); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-tmp_exp_name", 0, "define the exp_name (required)", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-tmp_inst", 0, "define the camera name (required)", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-tmp_telescope", 0, "define the telescope name (required)", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", 0); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-label", 0, "define a label (carried to chip stage)", NULL); + psMetadataAddBool(newExpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -newImfile + psMetadata *newImfileArgs = psMetadataAlloc(); + psMetadataAddS64(newImfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define the exp_id (required)", 0); + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-tmp_class_id", 0, "define the class ID (required)", NULL); + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-uri", 0, "define the URI (required)", NULL); + + // -updatenewExp + psMetadata *updatenewExpArgs = psMetadataAlloc(); + psMetadataAddS64(updatenewExpArgs, PS_LIST_TAIL, "-exp_id", 0, "define the exp_id (required)", 0); + psMetadataAddStr(updatenewExpArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-newExp", "", PXINJECT_MODE_NEWEXP, newExpArgs); + PXOPT_ADD_MODE("-newImfile", "", PXINJECT_MODE_NEWIMFILE, newImfileArgs); + PXOPT_ADD_MODE("-updatenewExp", "", PXINJECT_MODE_UPDATENEWEXP, updatenewExpArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxregister.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxregister.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxregister.c (revision 22235) @@ -0,0 +1,59 @@ +/* + * register.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxregister.h" + +bool pxnewExpSetState(pxConfig *config, psS64 exp_id, const char *state) +{ + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "0 is not a valid exp_id"); + return false; + } + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid newExp state: %s", state); + return false; + } + + char *query = "UPDATE newExp SET state = '%s' WHERE exp_id = %"PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_id %"PRId64, exp_id); + return false; + } + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxregister.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxregister.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxregister.h (revision 22235) @@ -0,0 +1,29 @@ +/* + * pxregister.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXREGISTER_H +#define PXREGISTER_H 1 + +#include + +#include "pxtools.h" + +bool pxnewExpSetState(pxConfig *config, psS64 exp_id, const char *state); + +#endif // PXREGISTER_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtag.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtag.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtag.c (revision 22235) @@ -0,0 +1,102 @@ +/* + * pxtag.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxtag.h" + +psString pxGenExpTag(pxConfig *config, const char *exp_name) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // start a transaction so we don't increment the expTag counter unless we + // can successfully retreive it's value + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + + if (!p_psDBRunQuery(config->dbh, + "UPDATE expTagCounter SET counter = LAST_INSERT_ID(counter + 1)")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + + // psDBLastInsertID() can't be used here as it called mysql_insert_id() + // which doesn't work with this trick. See: + // http://dev.mysql.com/doc/refman/4.1/en/information-functions.html + if (!p_psDBRunQuery(config->dbh, "SELECT LAST_INSERT_ID() as counter")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + // sanity check that we only got one row + if (psArrayLength(output) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have gotten 1 row but %lu rows were returned", psArrayLength(output)); + psFree(output); + return NULL; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return NULL; + } + + psMetadata *row = output->data[0]; + bool status = false; + psU64 counter = psMetadataLookupU64(&status, row, "counter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for counter"); + psFree(output); + return NULL; + } + psString exp_id = NULL; + psStringAppend(&exp_id, "%s.%" PRIu64, exp_name, counter); + psFree(output); + + return exp_id; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtag.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtag.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtag.h (revision 22235) @@ -0,0 +1,29 @@ +/* + * pxtag.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXTAG_H +#define PXTAG_H 1 + +#include + +#include "pxconfig.h" + +psString pxGenExpTag(pxConfig *config, const char *exp_name); + +#endif // PXTAG_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtools.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtools.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtools.c (revision 22235) @@ -0,0 +1,46 @@ +/* + * pxtool.c + * + * Copyright (C) 2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" + +bool pxIsValidState(const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // XXX replace strncmp with strcmp + + if (!strcmp(state, "new")) return true; + if (!strcmp(state, "reg")) return true; + if (!strcmp(state, "full")) return true; + if (!strcmp(state, "drop")) return true; + if (!strcmp(state, "wait")) return true; + if (!strcmp(state, "goto_cleaned")) return true; + if (!strcmp(state, "cleaned")) return true; + if (!strcmp(state, "update")) return true; + if (!strcmp(state, "purged")) return true; + if (!strcmp(state, "goto_purged")) return true; + + return false; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtools.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtools.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtools.h (revision 22235) @@ -0,0 +1,480 @@ +/* + * pxtools.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXTOOLS_H +#define PXTOOLS_H 1 + +#include +#include // for strcmp and strncmp +#include // for strcasecmp +#include // for unlink + +// #include +// #include +// #include + +#include +#include +#include + +#include "pxconfig.h" +#include "pxtoolsErrorCodes.h" + +#include "pxcam.h" +#include "pxchip.h" +#include "pxdata.h" +#include "pxfake.h" +#include "pxwarp.h" +#include "pxregister.h" +#include "pxtag.h" +#include "pxtree.h" + +# define MAX_ROWS 10e9 +# define PXTOOL_MODE_NONE 0x0 + +bool pxIsValidState(const char *state); + +bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code); + +psExit pxerrorGetExitStatus (); + +void pxUsage(FILE *stream, int argc, char **argv, const char *modeName, psMetadata *argSet); +bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets); + +#define PXOPT_ADD_MODE(option, comment, modeval, argset) \ +{ \ + if (!psMetadataAddMetadata(argSets, PS_LIST_TAIL, option, 0, comment, argset)) {;\ + psError(PS_ERR_UNKNOWN, false, "failed to add argset for %s", option); \ + } \ + psFree(argset); \ +\ + if (!psMetadataAddU32(modes, PS_LIST_TAIL, option, 0, comment, modeval)) {;\ + psError(PS_ERR_UNKNOWN, false, "failed to add argset for %s", option); \ + } \ +} + +#define PXOPT_LOOKUP_STR(var, md, key, required, ret) \ +psString var; \ +{ \ + bool status; \ + \ + var = psMetadataLookupStr(&status, md, key); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", key); \ + return ret; \ + } \ + \ + if (required && (!var)) { \ + psError(PS_ERR_UNKNOWN, true, "%s is required", key); \ + return ret; \ + } \ +} + +#define PXOPT_LOOKUP_F(var, md, key, type, required, ret) \ +ps##type var; \ +{ \ + bool status; \ + \ + var = psMetadataLookup##type(&status, md, key); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", key); \ + return ret; \ + } \ + \ + if (required && isnan(var)) { \ + psError(PS_ERR_UNKNOWN, true, "%s is required", key); \ + return ret; \ + } \ +} + +#define PXOPT_LOOKUP_F32(var, md, key, required, ret) \ + PXOPT_LOOKUP_F(var, md, key, F64, required, ret) + +#define PXOPT_LOOKUP_F64(var, md, key, required, ret) \ + PXOPT_LOOKUP_F(var, md, key, F64, required, ret) + +#define PXOPT_LOOKUP_PRIMITIVE(var, md, key, type, suffix, max, required, ret) \ +type var; \ +{ \ + psMetadataItem *item = psMetadataLookup(md, key); \ + if (!item) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", key); \ + return ret; \ + } \ + psAssert(item->comment, "metadata item comment should be defined"); \ + if (required && (!psStrcasestr(item->comment, "(found)"))) { \ + psError(PS_ERR_UNKNOWN, true, "%s is required", key); \ + return ret; \ + } \ + \ + var = item->data.suffix; \ + \ +} + +#define PXOPT_LOOKUP_S16(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psS16, S16, INT16_MAX, required, ret) + +#define PXOPT_LOOKUP_S32(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psS32, S32, INT32_MAX, required, ret) + +#define PXOPT_LOOKUP_S64(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psS64, S64, INT64_MAX, required, ret) + +#define PXOPT_LOOKUP_U16(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psU16, U16, UINT16_MAX, required, ret) + +#define PXOPT_LOOKUP_U32(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psU32, U32, UINT32_MAX, required, ret) + +#define PXOPT_LOOKUP_U64(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psU64, U64, UINT64_MAX, required, ret) + +#define PXOPT_LOOKUP_TIME(var, md, key, required, ret) \ +psTime *var; \ +{ \ + bool status; \ + \ + var = psMetadataLookupTime(&status, md, key); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", key); \ + return ret; \ + } \ + \ + if (required && (!var)) { \ + psError(PS_ERR_UNKNOWN, true, "%s is required", key); \ + return ret; \ + } \ +} + +#define PXOPT_LOOKUP_BOOL(var, md, key, ret) \ +bool var; \ +{ \ + bool status; \ + \ + var = psMetadataLookupBool(&status, md, key); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", key); \ + return ret; \ + } \ +} + +// XXX the PXOPT_COPY_* macros free 'to' on error + +#define PXOPT_COPY_PRIMITIVE(from, to, type, suffix, oldname, newname, newcomment) \ +{ \ + psMetadataItem *item = psMetadataLookup(from, oldname); \ + if (!item) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", oldname); \ + return false; \ + } \ + psAssert(item->comment, "metadata item comment should be defined"); \ +\ + if (psStrcasestr(item->comment, "(found)")) { \ + if (!psMetadataAdd##suffix(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, newcomment, item->data.suffix)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \ + psFree(to); \ + return false; \ + } \ + } \ +} + +#define PXOPT_COPY_V(from, to, type, suffix, oldname, newname, comment) \ +{ \ + bool status = false; \ + type var = psMetadataLookup##suffix(&status, from, oldname); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for " oldname); \ + return false; \ + } \ + if (var) { \ + if (!psMetadataAdd##suffix(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, comment, var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \ + psFree(to); \ + return false; \ + } \ + }\ +} + +#define PXOPT_COPY_F(from, to, type, oldname, newname, comment) \ +{ \ + bool status = false; \ + ps##type var = psMetadataLookup##type(&status, from, oldname); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for " oldname); \ + return false; \ + } \ + if (!isnan(var)) { \ + if (!psMetadataAdd##type(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, comment, var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \ + psFree(to); \ + return false; \ + } \ + } \ +} + +#define PXOPT_COPY_F32(from, to, oldname, newname, comment) \ + PXOPT_COPY_F(from, to, F32, oldname, newname, comment) + +#define PXOPT_COPY_F64(from, to, oldname, newname, comment) \ + PXOPT_COPY_F(from, to, F64, oldname, newname, comment) + +#define PXOPT_COPY_TIME(from, to, oldname, newname, comment) \ + PXOPT_COPY_V(from, to, psTime *, Time, oldname, newname, comment) + +#define PXOPT_COPY_STR(from, to, oldname, newname, comment) \ + PXOPT_COPY_V(from, to, psString, Str, oldname, newname, comment) + +#define PXOPT_COPY_S16(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psS16, S16, oldname, newname, comment) + +#define PXOPT_COPY_S32(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psS32, S32, oldname, newname, comment) + +#define PXOPT_COPY_S64(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psS64, S64, oldname, newname, comment) + +#define PXOPT_COPY_U16(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psU16, U16, oldname, newname, comment) + +#define PXOPT_COPY_U32(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psU32, U32, oldname, newname, comment) + +#define PXOPT_COPY_U64(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psU64, U64, oldname, newname, comment) + + +/*** these PXOPT_ADD_WHERE macros are used to construct the default sql elements ***/ + +#define PXOPT_ADD_WHERE_STR(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_STR_ALIAS(name,realname) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, realname, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S16(name) \ +{ \ + psS16 s16 = 0; \ + bool status = false; \ + if ((s16= psMetadataLookupS16(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS16(config->where, PS_LIST_TAIL, #name, 0, "==", s16)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_BOOL(name) \ +{ \ + bool value = false; \ + bool status = false; \ + if ((value = psMetadataLookupBool(&status, config->args, "-" #name))) { \ + if (!psMetadataAddBool(config->where, PS_LIST_TAIL, #name, 0, "==", value)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S32(name) \ +{ \ + psS32 s32 = 0; \ + bool status = false; \ + if ((s32= psMetadataLookupS32(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, #name, 0, "==", s32)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S64(name) \ +{ \ + psS64 s64 = 0; \ + bool status = false; \ + if ((s64= psMetadataLookupS64(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, #name, 0, "==", s64)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_F32(name) \ +{ \ + psF32 var = 0; \ + bool status = false; \ + if ((var = psMetadataLookupF32(&status, config->args, "-" #name))) { \ + if (!isnan(var)) { \ + if (!psMetadataAddF32(config->where, PS_LIST_TAIL, #name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_F64(name) \ +{ \ + psF64 var = 0; \ + bool status = false; \ + if ((var = psMetadataLookupF64(&status, config->args, "-" #name))) { \ + if (!isnan(var)) { \ + if (!psMetadataAddF64(config->where, PS_LIST_TAIL, #name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_BOOL_ALIAS(flag,name) \ +{ \ + bool value = 0; \ + bool status = false; \ + if ((value = psMetadataLookupBool(&status, config->args, flag))) { \ + if (!psMetadataAddBOOL(config->where, PS_LIST_TAIL, name, 0, "==", value)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S16_ALIAS(flag,name) \ +{ \ + psS16 s16 = 0; \ + bool status = false; \ + if ((s16= psMetadataLookupS16(&status, config->args, flag))) { \ + if (!psMetadataAddS16(config->where, PS_LIST_TAIL, name, 0, "==", s16)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S32_ALIAS(flag,name) \ +{ \ + psS32 s32 = 0; \ + bool status = false; \ + if ((s32= psMetadataLookupS32(&status, config->args, flag))) { \ + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, name, 0, "==", s32)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S64_ALIAS(flag,name) \ +{ \ + psS64 s64 = 0; \ + bool status = false; \ + if ((s64= psMetadataLookupS64(&status, config->args, flag))) { \ + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, name, 0, "==", s64)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_F32_ALIAS(flag,name) \ +{ \ + psF32 var = 0; \ + bool status = false; \ + if ((var = psMetadataLookupF32(&status, config->args, flag))) { \ + if (!isnan(var)) { \ + if (!psMetadataAddF32(config->where, PS_LIST_TAIL, name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_F64_ALIAS(flag,name) \ +{ \ + psF64 var = 0; \ + bool status = false; \ + if ((var = psMetadataLookupF64(&status, config->args, flag))) { \ + if (!isnan(var)) { \ + if (!psMetadataAddF64(config->where, PS_LIST_TAIL, name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_TIME_STR(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + psTime *time = psTimeFromISO(str, PS_TIME_UTC); \ + if (!time) { \ + psError(PS_ERR_UNKNOWN, false, "failed to convert " #name " into a psTime object"); \ + psFree(config); \ + return NULL; \ + } \ + psMetadataItem *item = psMetadataLookup(config->args, "-" #name); \ + if (item) { \ + str = item->comment; \ + } else { \ + str = NULL; \ + } \ + if (!psMetadataAddTime(config->where, PS_LIST_TAIL, #name, 0, str, time)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + psFree(time); \ + } \ +} + +#endif // PXTOOLS_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtoolsErrorCodes.c.in =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtoolsErrorCodes.c.in (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtoolsErrorCodes.c.in (revision 22235) @@ -0,0 +1,25 @@ +/* + * The line + { PXTOOLS_ERR_$X{ErrorCode}, "$X{ErrorDescription}"}, + * (without the Xs) + * will be replaced by values from errorCodes.dat + */ +#include "pslib.h" +#include "pxtoolsErrorCodes.h" + +void pxtoolsErrorRegister(void) +{ + static psErrorDescription errors[] = { + { PXTOOLS_ERR_BASE, "First value we use; lower values belong to psLib" }, + { PXTOOLS_ERR_${ErrorCode}, "${ErrorDescription}"}, + }; + static int nerror = PXTOOLS_ERR_NERROR - PXTOOLS_ERR_BASE; // number of values in enum + + for (int i = 0; i < nerror; i++) { + psErrorDescription *tmp = psAlloc(sizeof(psErrorDescription)); + *tmp = errors[i]; + psErrorRegister(tmp, 1); + psFree(tmp); /* it's on the internal list */ + } + nerror = 0; // don't register more than once +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtoolsErrorCodes.dat =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtoolsErrorCodes.dat (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtoolsErrorCodes.dat (revision 22235) @@ -0,0 +1,10 @@ +# +# This file is used to generate pxtoolsErrorCodes.h +# +BASE = 1100 First value we use; lower values belong to psLib +UNKNOWN Unknown PM error code +ARGUMENTS Incorrect arguments +SYS System error +CONFIG Problem in configure files +PROG Programming error +DATA invalid data Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtoolsErrorCodes.h.in =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtoolsErrorCodes.h.in (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtoolsErrorCodes.h.in (revision 22235) @@ -0,0 +1,18 @@ +#if !defined(PXTOOLS_ERROR_CODES_H) +#define PXTOOLS_ERROR_CODES_H +/* + * The line + * PXTOOLS_ERR_$X{ErrorCode}, + * (without the X) + * + * will be replaced by values from errorCodes.dat + */ +typedef enum { + PXTOOLS_ERR_BASE = 512, + PXTOOLS_ERR_${ErrorCode}, + PXTOOLS_ERR_NERROR +} pxtoolsErrorCode; + +void pxtoolsErrorRegister(void); + +#endif Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtree.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtree.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtree.c (revision 22235) @@ -0,0 +1,239 @@ +/* + * pxio.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxtree.h" + +static void pxNodeFree(pxNode *node) +{ + psFree(node->name); + psFree(node->parent); + psFree(node->children); + psFree(node->data); +} + +pxNode *pxNodeAlloc(const char *name) +{ + PS_ASSERT_PTR_NON_NULL(name, NULL); + + pxNode *node = psAlloc(sizeof(pxNode)); + + node->name = psStringCopy(name); + node->parent = NULL; + node->children = psListAlloc(NULL); + node->data = NULL; + + psMemSetDeallocator(node, (psFreeFunc) pxNodeFree); + + return node; +} + +pxNode *pxNodeAddParent(pxNode *node, pxNode *parent) +{ + // add this node to it's parent's children list + psListAdd(parent->children, PS_LIST_TAIL, node); + // and set the parent pointer + node->parent = psMemIncrRefCounter(parent); + + return node; +} + +pxNode *pxNodeAddChild(pxNode *node, pxNode *child) +{ + // add the child node to this nodes list of children + psListAdd(node->children, PS_LIST_TAIL, child); + // if the child already has a parent, release the ref count + if (child->parent) { + psFree(child->parent); + } + // [re]parent the child + child->parent = psMemIncrRefCounter(node); + + return node; +} + +pxNode *pxNodeAddData(pxNode *node, psPtr data) +{ + node->data = psMemIncrRefCounter(data); + + return node; +} + +void pxNodePrint(FILE *stream, pxNode *node) +{ + fprintf(stream, "node name: %s\n", node->name); + fprintf(stream, "node parent's name: %s\n", node->parent + ? node->parent->name + : NULL); + psListIterator *iter = psListIteratorAlloc(node->children, 0, false); + pxNode *child = NULL; + while ((child = psListGetAndIncrement(iter))) { + fprintf(stream, "node child's name: %s\n", child->name); + } + psFree(iter); +} + + +static bool printNodes(void *arg, pxNode *node) +{ + pxNodePrint((FILE *)arg, node); + fprintf((FILE *)arg, "\n\n"); + + return true; +} + + +void pxTreePrint(FILE *stream, pxNode *root) +{ + pxTreeCrawl(root, printNodes, (void *)stream); +} + +// func() returning false means decend no futher along this branch + +bool pxTreeCrawl(pxNode *node, pxNodeFunc func, void *arg) +{ + // if func() returns false, stop + if (!func(arg, node)) { + return false; + } + + psListIterator *iter = psListIteratorAlloc(node->children, 0, false); + pxNode *child = NULL; + while ((child = psListGetAndIncrement(iter))) { + pxTreeCrawl(child, func, arg); + } + psFree(iter); + + return true; +} + +bool pxNodeFuncCountDependants(void *arg, pxNode *node) +{ + (*(int *)arg)++; + return true; +} + +bool pxNodeHasChildren(pxNode *node) +{ + int children = psListLength(node->children); + + return children ? true : false; +} + +bool pxNodeHasGrandChildren(pxNode *node) +{ + // find out how many nodes there are lower in the tree + // subtract the parent node from the count so we have just a tally of + // decendants + int nodes = -1; + pxTreeCrawl(node, pxNodeFuncCountDependants, &nodes); + psTrace("pxtree", PS_LOG_INFO, "node %s has %d dependants", node->name, nodes); + + // find out how many children this node has + int children = psListLength(node->children); + psTrace("pxtree", PS_LOG_INFO, "node %s has %d children", node->name, children); + + if (!children) { + // no children + return false; + } + + if (nodes < children) { + psAbort("you can't have fewer children than decendants!"); + } + + if (nodes == children) { + // no grandchildren + return false; + } + + return true; +} + +pxNode *pxTreeFromMetadata(psMetadata *md) +{ + psHash *forest = psHashAlloc(10); + + psMetadataIterator *iter = psMetadataIteratorAlloc(md, 0, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement(iter))) { + if (!(item->type == PS_DATA_STRING)) { + continue; + } + + // add this node to the forest and hopefully it'll get attached to the + // root tree + pxTreeBuilder(forest, item->name, item->data.str, NULL); + } + psFree(iter); + + pxNode *root = psHashLookup(forest, "root"); + psFree(forest); + + return psMemIncrRefCounter(root); +} + +psHash *pxTreeBuilder(psHash *forest, + const char *nodeName, + const char *childName, + psPtr data) +{ + // try to find a node with this name + pxNode *node = psHashLookup(forest, nodeName); + if (!node) { + // create a new node with this name + node = pxNodeAlloc(nodeName); + if (data) { + pxNodeAddData(node, data); + } + // add it to the hash of nodes + psHashAdd(forest, nodeName, node); + // node may be used below because it will still have a ref count of + // 1 from being on the hash, this is equivalent to the "view" we + // get from a hashlookup + psFree(node); + } else if (!node->data) { + pxNodeAddData(node, data); + } + + // does this node declare a child? + if (childName) { + // try to find a node with this name + pxNode *child = psHashLookup(forest, childName); + if (!child ) { + // create a new node with this name + child = pxNodeAlloc(childName); + // add it to the hash of nodes + psHashAdd(forest, childName, child); + // child may be used below because it will still have a ref + // count of 1 from being on the hash, this is equivalent to the + // "view" we get from a hashlookup + psFree(child); + } + pxNodeAddChild(node, child); + } + + return forest; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtree.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtree.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxtree.h (revision 22235) @@ -0,0 +1,72 @@ +/* + * pxtree.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXTREE_H +#define PXTREE_H 1 + +#include +#include + +#include + + +typedef struct pxNode { + char *name; + struct pxNode *parent; + psList *children; + void *data; +} pxNode; + +typedef bool (*pxNodeFunc)(void *arg, pxNode *node); + +pxNode *pxNodeAlloc( + const char *name +) PS_ATTR_MALLOC; + +pxNode *pxNodeAddParent(pxNode *node, pxNode *parent); +pxNode *pxNodeAddChild(pxNode *node, pxNode *child); +pxNode *pxNodeAddData(pxNode *node, psPtr data); + +void pxTreePrint(FILE *stream, pxNode *root); + +void pxNodePrint( + FILE *stream, + pxNode *node +); + +bool pxTreeCrawl( + pxNode *node, + pxNodeFunc func, + void *arg +); + +bool pxNodeFuncCountDependants(void *arg, pxNode *node); + +bool pxNodeHasChildren(pxNode *node); + +bool pxNodeHasGrandChildren(pxNode *node); + +pxNode *pxTreeFromMetadata(psMetadata *md); + +psHash *pxTreeBuilder(psHash *forest, + const char *nodeName, + const char *childName, + psPtr data); + +#endif // PXTREE_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxwarp.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxwarp.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxwarp.c (revision 22235) @@ -0,0 +1,157 @@ +/* + * pxwarp.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxwarp.h" + +bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid warpRun state: %s", state); + return false; + } + + char *query = "UPDATE warpRun SET state = '%s' WHERE warp_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, warp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for warp_id %" PRId64, warp_id); + return false; + } + + return true; +} + + +bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid chipRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + if (whereClause && strlen(whereClause) > 0) { + psStringAppend(&query, " %s", whereClause); + } + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxwarpRunSetLabel(pxConfig *config, psS64 warp_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE warpRun SET warpRun.label = '%s' WHERE warp_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, label, warp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change label for warp_id %" PRId64, warp_id); + return false; + } + + return true; +} + + +bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxwarpQueueByFakeID(pxConfig *config, + psS64 fake_id, + char *workdir, + char *label, + char *dvodb, + char *tess_id, + char *end_stage) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // depend on the f-keys to make sure we have a valid cam_id + if (!warpRunInsert(config->dbh, + 0, // ID + fake_id, + "warp", // mode + "new", // state + workdir, + "dirty", // workdir_state + label, + dvodb, + tess_id, + end_stage, + NULL, // registered + false // magiced + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return true; + } + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pxwarp.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pxwarp.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pxwarp.h (revision 22235) @@ -0,0 +1,40 @@ +/* + * pxwarp.h + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXWARP_H +#define PXWARP_H 1 + +#include + +#include "pxtools.h" + +bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state); +bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxwarpRunSetLabel(pxConfig *config, psS64 warp_id, const char *label); +bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +bool pxwarpQueueByFakeID(pxConfig *config, + psS64 fake_id, + char *workdir, + char *label, + char *dvodb, + char *tess_id, + char *end_stage); + +#endif // PXWARP_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetexp.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetexp.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetexp.c (revision 22235) @@ -0,0 +1,368 @@ +/* + * pzgetexp.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pzgetexp.h" + +#define PRODUCT_LS_CMD "dsproductls" + +static bool go (pxConfig *config); +static psArray *parseFileSets(pxConfig *config, const char *str); + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pzgetexpConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + if (!go(config)) { + goto FAIL; + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool go(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // required + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + //optional + PXOPT_LOOKUP_S32(timeout, config->args, "-timeout", false, false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + // find last fileset/exp_name (if we have one) + bool haveLastFileSet = false; + psString lastFileSet = NULL; + + // -all means "request all known filesets" + if (!all) { + char *query = "SELECT * from summitExp WHERE camera = \"%s\" and TELESCOPE = \"%s\" ORDER BY dateobs DESC LIMIT 1"; + if (!p_psDBRunQuery(config->dbh, query, camera, telescope)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "no summitExp rows found"); + haveLastFileSet = false; + psFree(output); + } else { + haveLastFileSet = true; + bool status = false; + lastFileSet = psStringCopy(psMetadataLookupStr(&status, output->data[0], "exp_name")); + psFree(output); + } + } + + // invoke dsproductls + // dsproductls --uri <> --last_fileset <> + psString cmd = NULL; + if (haveLastFileSet) { + psStringAppend(&cmd, "%s --uri %s --last_fileset %s", + PRODUCT_LS_CMD, uri, lastFileSet); + psFree(lastFileSet); + } else { + psStringAppend(&cmd, "%s --uri %s", PRODUCT_LS_CMD, uri); + } + if (timeout) { + psStringAppend(&cmd, " --timeout %d", timeout); + } + + psTrace("pzgetexp", PS_LOG_INFO, "cmd is: %s\n", cmd); + + FILE *output = popen(cmd, "r"); + psFree(cmd); + + if (!output) { + psError(PS_ERR_UNKNOWN, true, "popen() failed"); + return false; + } + psString cmdOutput = psSlurpFile(output); + pclose(output); + + psArray *newSummitExps = parseFileSets(config, cmdOutput); + psFree(cmdOutput); + if (!newSummitExps) { + // XXX not nessicarily an error + psError(PS_ERR_UNKNOWN, true, "no new fileSet/exp IDs"); + return false; + } + if (!psArrayLength(newSummitExps)) { + psTrace("pzgetexp", PS_LOG_INFO, "no new fileSet/exp IDs"); + psFree(newSummitExps); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } + + // increase memory table size limits; deafult is 16MB + { + // 512MB + char *query = "SET max_heap_table_size = 1024*1024*512"; + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } + } + + // create a temporry table + { + char *query = "CREATE TEMPORARY TABLE incoming" + " (exp_name VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), dateobs DATETIME, exp_type VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_name, camera, telescope))" + " ENGINE=MEMORY"; + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } + } + + { + char *query = "INSERT INTO incoming (exp_name, camera, telescope, dateobs, exp_type, uri) VALUES (?, ?, ?, ?, ?, ?)"; + + long inserted = p_psDBRunQueryPrepared(config->dbh, newSummitExps, query); + if (inserted < 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } + // sanity check that we actually inserted something + if (inserted == 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error -- we should have inserted at least one row"); + psFree(newSummitExps); + return false; + } + } + + psFree(newSummitExps); + + // add new exps to summitExp + { + char *query = + "INSERT INTO summitExp" + " SElECT" + " incoming.*," + " NULL," // imfiles + " 0," // fault + " NULL" // epoch + " FROM incoming" + " LEFT JOIN summitExp" + " USING(exp_name, camera, telescope)" + " WHERE" + " summitExp.exp_name is NULL" + " AND summitExp.camera is NULL" + " AND summitExp.telescope is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psArray *parseFileSets(pxConfig *config, const char *str) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(str, NULL); + + // these are constants for all records parsed -- look them up before we do + // any work + // required + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + + // split the string into lines + psList *doc = psStringSplit(str, "\n", false); + + psListIterator *lineCursor = psListIteratorAlloc(doc, 0, false); + + psArray *summitExps = psArrayAllocEmpty(psListLength(doc)); + psString line; + while ((line = psListGetAndIncrement(lineCursor))) { + psTrace("pzgetimfile", PS_LOG_INFO, "parsing line: %s\n", line); + + // split line into tokens + psList *tokens = psStringSplit(line, " ", false); + + // check to see if this line is a comment (or if the first token is + // NULL) + if (!psListGet(tokens, 0) || *((char *)psListGet(tokens, 0)) == '#') { + psFree(tokens); + continue; + } + + // check that we have the right number of tokens + // print "# uri fileset datetime type\n"; + if (psListLength(tokens) != 4) { + // error + return false; + } + + // find the values of interest + psListIterator *tokenCursor = psListIteratorAlloc(tokens, 0, false); + char *uri = psListGetAndIncrement(tokenCursor); + char *exp_name = psListGetAndIncrement(tokenCursor); // fileset + char *dateobsStr= psListGetAndIncrement(tokenCursor); // datetime + char *exp_type = psListGetAndIncrement(tokenCursor); // type + + // create a new metadata to represent this line and it's values + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_name", 0, NULL, exp_name)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_name"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "camera", 0, NULL, camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "telescope", 0, NULL, telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + psTime *dateobs = psTimeFromISO(dateobsStr, PS_TIME_UTC); + if (!psMetadataAddTime(md, PS_LIST_TAIL, "dateobs", 0, NULL, dateobs)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(dateobs); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + psFree(dateobs); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_type", 0, NULL, exp_type)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_type"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "uri", 0, NULL, uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + + // must be freed after the new metadata is built -- holds the strings + psFree(tokenCursor); + psFree(tokens); + + // add the new metadata to the result set + psArrayAdd(summitExps, 0, md); + + // debugging + if (psTraceGetLevel("pzgetexp") == PS_LOG_INFO) { + psString doc = psMetadataConfigFormat(md); + psTrace("pzgetexp", PS_LOG_INFO, "parsed line as:\n %s\n", doc); + psFree(doc); + } + + psFree(md); + + } + + psFree(lineCursor); + psFree(doc); + + return summitExps; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetexp.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetexp.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetexp.h (revision 22235) @@ -0,0 +1,27 @@ +/* + * pzgetexp.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PZGETEXP_H +#define PZGETEXP_H 1 + +#include "pxtools.h" + +pxConfig *pzgetexpConfig(pxConfig *config, int argc, char **argv); + +#endif // PZGETEXP_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetexpConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetexpConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetexpConfig.c (revision 22235) @@ -0,0 +1,88 @@ +/* + * pzgetexpConfig.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" + +pxConfig *pzgetexpConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + psMetadata *args = psMetadataAlloc(); + psMetadataAddStr(args , PS_LIST_TAIL, "-uri", 0, + "DataStore product URI (required)", ""); + psMetadataAddStr(args , PS_LIST_TAIL, "-inst", 0, + "camera name (required)", ""); + psMetadataAddStr(args , PS_LIST_TAIL, "-telescope", 0, + "telescope name (required)", ""); + psMetadataAddS32(args, PS_LIST_TAIL, "-timeout", 0, + "HTTP timeout", 0); + psMetadataAddBool(args, PS_LIST_TAIL, "-all", 0, + "download ALL filesets", 0); + + + bool status = false; + if (!psArgumentParse(args, &argc, argv) + || argc != 1 + || strcmp(psMetadataLookupStr(&status, args, "-uri"), "") == 0 + || strcmp(psMetadataLookupStr(&status, args, "-inst"), "") == 0 + || strcmp(psMetadataLookupStr(&status, args, "-telescope"), "") == 0 + ) { + fprintf(stderr, "error parsing arguments\n"); + printf("\nPan-STARRS Phase Z Get Exposures Tool\n"); + printf("Usage: %s -uri -inst -telescope [-timeout ]\n\n", + argv[0]); + psArgumentHelp(args); + psFree(config); + return NULL; + } + + config->args = args; + // don't free args here as it's silly to increment the ref count then + // "free" it + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if(!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't connect to db\n"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetimfiles.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetimfiles.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetimfiles.c (revision 22235) @@ -0,0 +1,482 @@ +/* + * pzgetimfiles.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtag.h" +#include "pxtools.h" +#include "pzgetimfiles.h" + +#define FILESET_LS_CMD "dsfilesetls" + +static bool go (pxConfig *config); +static psArray *parseFiles(pxConfig *config, const char *str); + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pzgetimfilesConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + if (!go(config)) { + goto FAIL; + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool go(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // required + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_STR(filesetid, config->args, "-filesetid", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + + // optional + PXOPT_LOOKUP_S32(timeout, config->args, "-timeout", false, false); + + // invoke dsfilesetls + psString cmd = NULL; + psStringAppend(&cmd, "%s --uri %s", FILESET_LS_CMD, uri); + if (timeout) { + psStringAppend(&cmd, " --timeout %d", timeout); + } + + psTrace("pzgetimfiles", PS_LOG_INFO, "cmd is: %s\n", cmd); + + FILE *output = popen(cmd, "r"); + psFree(cmd); + + if (!output) { + psError(PS_ERR_UNKNOWN, true, "popen() failed"); + return false; + } + + psString cmdOutput = psSlurpFile(output); + int status = pclose(output); + + if (status != 0) { + // mark the summitExp row as faulted + if (!p_psDBRunQuery(config->dbh, "UPDATE summitExp SET fault = %d WHERE exp_name = '%s' AND camera = '%s' AND telescope = '%s'", WEXITSTATUS(status), filesetid, camera, telescope)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + + psError(PS_ERR_UNKNOWN, true, "%s failed with exit status %d", + FILESET_LS_CMD, WEXITSTATUS(status)); + psFree(cmdOutput); + return false; + } + + // prase output of dsfilesetls + psArray *newImfiles = parseFiles(config, cmdOutput); + if (!newImfiles) { + // XXX not nessicarily an error + psError(PS_ERR_UNKNOWN, true, "no new files/imfiles"); + psFree(cmdOutput); + return false; + } + psFree(cmdOutput); + + // save the number of new Imfiles; + long imfiles = psArrayLength(newImfiles); + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // if the fileset was empty (no files) then we can bail out early + if (imfiles == 0) { + psFree(newImfiles); + + char *query = + "UPDATE summitExp" + " SET imfiles = %d" + " WHERE exp_name = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, imfiles, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // remove the pzDownloadExp entry for this exp (fileset) + { + char *query = + "DELETE FROM pzDownloadExp" + " WHERE" + " exp_name = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have removed only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; + } + + // create a temp table + { + char *query = + "CREATE TEMPORARY TABLE incoming (exp_name VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), file_id VARCHAR(64), bytes INT, md5sum VARCHAR(32), class VARCHAR(64), class_id VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_name, camera, telescope, class, class_id)) ENGINE=MEMORY"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); + return false; + } + } + + // load the imfiles (files) into the temp table + { + char *query = "INSERT INTO incoming (exp_name, camera, telescope, file_id, bytes, md5sum, class, class_id, uri) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; + + long inserted = p_psDBRunQueryPrepared(config->dbh, newImfiles, query); + if (inserted < 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); + return false; + } + // sanity check that we actually inserted something + if (inserted == 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error -- we should have inserted at least one row"); + psFree(newImfiles); + return false; + } + } + + psFree(newImfiles); + + // copy imfiles (files) from the temp table into summitImfiles + { + char *query = + "INSERT IGNORE INTO summitImfile" + " SELECT" + " incoming.exp_name," + " incoming.camera," + " incoming.telescope," + " incoming.file_id," + " incoming.bytes," + " incoming.md5sum," + " incoming.class," + " incoming.class_id," + " incoming.uri," + " NULL" // epoch + " FROM incoming"; + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // update summitExp.imfiles (should have been NULL) + { + char *query = + "UPDATE summitExp" + " SET imfiles = (SELECT COUNT(*) FROM summitImfile" + " WHERE" + " exp_name = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'" + ")" + " WHERE" + " exp_name = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'" + " AND imfiles IS NULL"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have updated only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // add new exps to pzDownloadExp -- must be done before the new exps are + // added to summitExp because of the SQL logic + { + char *query = + "INSERT IGNORE INTO pzDownloadExp" + " SELECT" + " incoming.exp_name," + " incoming.camera," + " incoming.telescope," + " \"run\"," // state + " NULL" + " FROM incoming" + " GROUP BY" + " incoming.exp_name," + " incoming.camera," + " incoming.telescope"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psArray *parseFiles(pxConfig *config, const char *str) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(str, NULL); + + // these are constants for all records parsed -- look them up before we do + // any work + PXOPT_LOOKUP_STR(exp_name, config->args, "-filesetid", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + + // split the string into lines + psList *doc = psStringSplit(str, "\n", false); + + psListIterator *lineCursor = psListIteratorAlloc(doc, 0, false); + + psArray *pzPendingImfiles = psArrayAllocEmpty(psListLength(doc)); + psString line; + while ((line = psListGetAndIncrement(lineCursor))) { + psTrace("pzgetimfiles", PS_LOG_INFO, "parsing line: %s\n", line); + + // split line into tokens + psList *tokens = psStringSplit(line, " ", false); + + // check to see if this line is a comment (or if the first token is + // NULL) + if (!psListGet(tokens, 0) || *((char *)psListGet(tokens, 0)) == '#') { + psFree(tokens); + continue; + } + + // check that we have the right number of tokens + // print "# uri fileid bytes md5sum type \n"; + if (!psListLength(tokens) > 5) { + psError(PS_ERR_UNKNOWN, true, "invalid line format: %s", line); + psFree(tokens); + psFree(pzPendingImfiles); + psFree(lineCursor); + psFree(doc); + return false; + } + + // find the values of interest + psListIterator *tokenCursor = psListIteratorAlloc(tokens, 0, false); + char *uri = psListGetAndIncrement(tokenCursor); + char *file_id = psListGetAndIncrement(tokenCursor); // fileid + char *bytes = psListGetAndIncrement(tokenCursor); // bytes + char *md5sum = psListGetAndIncrement(tokenCursor); // md5sum + char *class = psListGetAndIncrement(tokenCursor); // type + char *class_id = psListGetAndIncrement(tokenCursor); // chipname + + // create a new metadata to represent this line and it's values + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_name", 0, NULL, exp_name)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_name"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "camera", 0, NULL, camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "telescope", 0, NULL, telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "file_id", 0, NULL, file_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bytes"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddS64(md, PS_LIST_TAIL, "bytes", 0, NULL, (psS64)atoll(bytes))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bytes"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "md5sum", 0, NULL, md5sum)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item md5sum"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "class", 0, NULL, class)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "class_id", 0, NULL, class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "uri", 0, NULL, uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + + // must be freed after the new metadata is built -- holds the strings + psFree(tokenCursor); + psFree(tokens); + + // debugging + if (psTraceGetLevel("pzgetimfiles") >= PS_LOG_INFO) { + psString doc = psMetadataConfigFormat(md); + psTrace("pzgetimfiles", PS_LOG_INFO, "parsed line as:\n %s\n", doc); + psFree(doc); + } + + psArrayAdd(pzPendingImfiles, 0, md); + + psFree(md); + } + + psFree(lineCursor); + psFree(doc); + + if (!psArrayLength(pzPendingImfiles)) { + psError(PS_ERR_UNKNOWN, false, "string contained no valid rows"); + psFree(pzPendingImfiles); + return false; + } + + return pzPendingImfiles; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetimfiles.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetimfiles.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetimfiles.h (revision 22235) @@ -0,0 +1,27 @@ +/* + * pzgetimfiles.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PZGETIMFILES_H +#define PZGETIMFILES 1 + +#include "pxtools.h" + +pxConfig *pzgetimfilesConfig(pxConfig *config, int argc, char **argv); + +#endif // PZGETIMFILES_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetimfilesConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetimfilesConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pzgetimfilesConfig.c (revision 22235) @@ -0,0 +1,89 @@ +/* + * pzgetimfilesConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" + +pxConfig *pzgetimfilesConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + psMetadata *args = psMetadataAlloc(); + psMetadataAddStr(args , PS_LIST_TAIL, "-uri", 0, + "DataStore FileSet URI (required)", NULL); + psMetadataAddStr(args , PS_LIST_TAIL, "-filesetid", 0, + "FileSet ID (required)", NULL); + psMetadataAddStr(args, PS_LIST_TAIL, "-inst", 0, + "Camera ID (required)", NULL); + psMetadataAddStr(args, PS_LIST_TAIL, "-telescope", 0, + "Telescope ID (required)", NULL); + psMetadataAddS32(args, PS_LIST_TAIL, "-timeout", 0, + "HTTP timeout", 0); + + + bool status = false; + if (!psArgumentParse(args, &argc, argv) + || argc != 1 + || (psMetadataLookupStr(&status, args, "-uri") == NULL) + || (psMetadataLookupStr(&status, args, "-filesetid") == NULL) + || (psMetadataLookupStr(&status, args, "-inst") == NULL) + || (psMetadataLookupStr(&status, args, "-telescope") == NULL) + ) { + fprintf(stderr, "error parsing arguments\n"); + printf("\nPan-STARRS Phase Z Get Imfiles Tool\n"); + printf("Usage: %s -uri -filesetid -inst -telescope ]\n\n", argv[0]); + psArgumentHelp(args); + psFree(args); + psFree(config); + return NULL; + } + + config->args = args; + // don't free args here as it's silly to increment the ref count then + // "free" it + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if(!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't connect to db\n"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pztool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pztool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pztool.c (revision 22235) @@ -0,0 +1,935 @@ +/* + * pztool.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pslib.h" +#include "pxtools.h" +#include "pxdata.h" +#include "pztool.h" + +static bool adddatastoreMode(pxConfig *config); +static bool datastoreMode(pxConfig *config); +static bool seenMode(pxConfig *config); +static bool pendingExpMode(pxConfig *config); +static bool pendingImfileMode(pxConfig *config); + +static bool copydoneMode(pxConfig *config); +static bool copiedMode(pxConfig *config); +static bool updatecopiedMode(pxConfig *config); +static bool revertcopiedMode(pxConfig *config); + +static bool clearcommonfaultsMode(pxConfig *config); +static bool advanceMode(pxConfig *config); + +static bool copydoneCompleteExp(pxConfig *config); +static psArray *pzGetPendingCameras(pxConfig *config); +static psArray *pzArrayZip(psArray *arraySet, psS64 limit); +static bool pzDownloadExpSetState(pxConfig *config, const char *exp_name, const char *camera, const char *telescope, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pztoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(PZTOOL_MODE_ADDDATASTORE, adddatastoreMode); + MODECASE(PZTOOL_MODE_DATASTORE, datastoreMode); + MODECASE(PZTOOL_MODE_SEEN, seenMode); + MODECASE(PZTOOL_MODE_PENDINGEXP, pendingExpMode); + MODECASE(PZTOOL_MODE_PENDINGIMFILE, pendingImfileMode); + MODECASE(PZTOOL_MODE_COPYDONE, copydoneMode); + MODECASE(PZTOOL_MODE_COPIED, copiedMode); + MODECASE(PZTOOL_MODE_UPDATECOPIED, updatecopiedMode); + MODECASE(PZTOOL_MODE_REVERTCOPIED, revertcopiedMode); + MODECASE(PZTOOL_MODE_CLEARCOMMONFAULTS, clearcommonfaultsMode); + MODECASE(PZTOOL_MODE_ADVANCE, advanceMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool adddatastoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + if (!pzDataStoreInsert(config->dbh, + camera, + telescope, + uri, + NULL // epoch + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool datastoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!p_psDBRunQuery(config->dbh, "SELECT * FROM pzDataStore")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pzDataStore", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool seenMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM summitExp"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "summitExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "summitExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingExpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + + PXOPT_LOOKUP_BOOL(desc, config->args, "-desc", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // XXX leave this query here ? + psString query = psStringCopy( + "SELECT" + " summitExp.*" + " FROM summitExp" + " LEFT JOIN pzDownloadExp" + " USING(exp_name, camera, telescope)" + " WHERE" + " pzDownloadExp.exp_name IS NULL" + " AND pzDownloadExp.camera IS NULL" + " AND pzDownloadExp.telescope IS NULL" + " AND summitExp.fault = 0" + ); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pzDownloadExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, " ORDER BY summitExp.dateobs"); + if (desc) { + psStringAppend(&query, " DESC"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pzDownloadExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingImfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + + PXOPT_LOOKUP_BOOL(desc, config->args, "-desc", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psArray *cameras = pzGetPendingCameras(config); + if (!cameras) { + psError(PXTOOLS_ERR_DATA, false, "failed to find any cameras"); + return false; + } + + // array to hold the aggregate results + psArray *cameraImfiles = psArrayAlloc(0); + + for (long i = 0; i < psArrayLength(cameras); i++) { + psString query = pxDataGet("pztool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(cameraImfiles); + return false; + } + + bool status; + psString camera = psMetadataLookupStr(&status, cameras->data[i], "camera"); + psStringAppend(&query, " WHERE camera = \"%s\"", camera); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pzDownloadImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psStringAppend(&query, " ORDER BY dateobs"); + if (desc) { + psStringAppend(&query, " DESC"); + } + + // request the full "limit" from each known camera and throw away any + // "extra" rows that we may have after merging the results. This is + // a lot simplier than a complicated scheme (tried that) to attempt to + // request on the right number of rows for each camera + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(cameraImfiles); + return false; + } + psFree(query); + + psArray *result = p_psDBFetchResult(config->dbh); + if (!result) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(cameraImfiles); + return false; + } + if (!psArrayLength(result)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(result); + continue; + } + + // add this query into the array of result set + psArrayAdd(cameraImfiles, 0, result); + psFree(result); + } + psFree(where); + + // stitch the arrays of imfiles together + psArray *output = pzArrayZip(cameraImfiles, limit); + psFree(cameraImfiles); + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pzDownloadImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool copydoneMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(class, config->args, "-class", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_BOOL(row_lock, config->args, "-row_lock", false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // NOTE : the rest of the command-line args are parsed in copydoneCompleteExp + + // start a transaction early so it will contain any row level locks + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // query to get an excluse lock on this exposure in + // pzDownloadExp + psString lock_query = NULL; + if (row_lock) { + lock_query = psStringCopy("SELECT * FROM pzDownloadExp"); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&lock_query, " %s FOR UPDATE", whereClause); + psFree(whereClause); + } + psFree(where); + + // aquire a lock on the pzDownloadExp record + // lock persists until the transaction is committed + if (!p_psDBRunQuery(config->dbh, lock_query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(lock_query); + return false; + } + psFree(lock_query); + + // we must fetch the result set from aquiring the row level lock or + // MySQL will barf all over us. + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(output); + } + + if (!pzDownloadImfileInsert(config->dbh, + exp_name, + camera, + telescope, + class, + class_id, + uri, + code, + NULL, // epoch + hostname + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!copydoneCompleteExp(config)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool copydoneCompleteExp(pxConfig *config) +{ + // THIS FUNCTION MUST BE INVOKED FROM INSIDE A TRANSACTION!!! + + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX this is an ugly hack! + // we are passing exp level info to a imfile level mode (-copydone) + // these options are thrown away unless we just -copydone'd the last imfile + // in an exp. + + // optional + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + // find all exposures that have had all of their imfiles downloaded but do + // not appear in newExp + psString query = pxDataGet("pztool_find_completed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // find completed exps + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + pzDownloadExpRow *doneExp = pzDownloadExpObjectFromMetadata(row); + if (!doneExp) { + psError(PS_ERR_UNKNOWN, false, "pzDownloadExpObjectFromMetadata() failed"); + psFree(doneExp); + psFree(output); + return false; + } + + if (!newExpInsert(config->dbh, + 0x0, // exp_id + doneExp->exp_name, // tmp_exp_name + doneExp->camera, // tmp_camera + doneExp->telescope, // tmp_telescope + "run", // state + workdir, // workdir + "dirty", // workdir state + NULL, // reduction class + dvodb, // dvodb + tess_id, // tess_id + end_stage, // end_stage + label, + NULL // epoch + ) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(doneExp); + psFree(output); + return false; + } + + psS64 exp_id = psDBLastInsertID(config->dbh); + + // insert newImfiles + { + char *query = + "INSERT INTO newImfile" + " SElECT" + " %" PRId64 "," // exp_id + " pzDownloadImfile.class_id," // tmp_class_id + " pzDownloadImfile.uri," // uri + " NULL" // epoch + " FROM pzDownloadImfile" + " WHERE" + " pzDownloadImfile.exp_name = '%s'" + " AND pzDownloadImfile.camera = '%s'" + " AND pzDownloadImfile.telescope = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, exp_id, doneExp->exp_name, doneExp->camera, doneExp->telescope)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(doneExp); + psFree(output); + return false; + } + + // sanity check: we should have inserted at least one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row but %" PRIu64 " rows were modified", affected); + psFree(doneExp); + psFree(output); + return false; + } + } + + // set pzDownloadExp.state to 'stop' + if (!pzDownloadExpSetState(config, doneExp->exp_name, doneExp->camera, doneExp->telescope, "stop")) { + psError(PS_ERR_UNKNOWN, false, "failed to change pzDownloadExp.state for %s:%s:%s", doneExp->exp_name, doneExp->camera, doneExp->telescope); + psFree(doneExp); + psFree(output); + return false; + } + + psFree(doneExp); + } + + psFree(output); + + return true; +} + +static psArray *pzGetPendingCameras(pxConfig *config) +{ + // get a list of cameras we've seen exps for + if (!p_psDBRunQuery(config->dbh, "SELECT DISTINCT camera FROM pzDownloadExp")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *cameras = p_psDBFetchResult(config->dbh); + if (!cameras) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + if (!psArrayLength(cameras)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(cameras); + return psArrayAlloc(0); + } + + return cameras; +} + +static psArray *pzArrayZip(psArray *arraySet, psS64 limit) +{ + // figure out the combined size of all arrays in the set + long setSize = 0; + for (long i = 0; i < psArrayLength(arraySet); i++) { + setSize += psArrayLength(arraySet->data[i]); + } + + // treat 0 as "no limit" + if (limit == 0) { + limit = setSize; + } + + psArray *output = psArrayAllocEmpty(limit); + // loop over each array in the set forever + for ( + // init + long counter = 0, // the total number of elements zipped so far + i = 0, // which array in the set + index = 0; // the depth into each array + // test + (counter < setSize) + && (counter < limit) + && (i < psArrayLength(arraySet)); + // incr + counter++, ++i, + i = i % psArrayLength(arraySet), + i % psArrayLength(arraySet) ? : ++index + ) { + + psArray *array = arraySet->data[i]; + // make sure that this array has not run out of elements + if (!(index < psArrayLength(array))) { + continue; + } + + psArrayAdd(output, 0, array->data[index]); + } + + return output; +} + + +static bool copiedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = psStringCopy("SELECT * FROM pzDownloadImfile"); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "WHERE pzDownloadImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "WHERE pzDownloadImfile.fault = 0"); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pzDownloadImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool updatecopiedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + if (!pxSetFaultCode(config->dbh, "pzDownloadImfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree(where); + + return true; +} + + +static bool revertcopiedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("pztool_revertcopied.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pzDownloadImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool clearcommonfaultsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + +{ + psString query = pxDataGet("pztool_revert_downloadimfile_faults.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); +} + +{ + psString query = pxDataGet("pztool_revert_fileset_faults.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); +} + + return true; +} + +static bool advanceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // NOTE : the command-line args are parsed in copydoneCompleteExp + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!copydoneCompleteExp(config)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + return true; +} + + +static bool pzDownloadExpSetState(pxConfig *config, const char *exp_name, const char *camera, const char *telescope, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid pzDownloadExp state: %s", state); + return false; + } + + char *query = "UPDATE pzDownloadExp SET state = '%s' WHERE exp_name = '%s' and camera = '%s' and telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, state, exp_name, camera, telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for %s:%s:%s", exp_name, camera, telescope); + return false; + } + + return true; +} + + +#if 0 +static psArray *pzArrayAddArray(psArray *array, psArray *input) +{ + for (long i = 0; i < psArrayLength(input); i++) { + psArrayAdd(array, psArrayLength(input), input->data[i]); + } + + return array; +} +#endif Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pztool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pztool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pztool.h (revision 22235) @@ -0,0 +1,42 @@ +/* + * pztool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PZTOOL_H +#define PZTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + PZTOOL_MODE_NONE = 0x0, + PZTOOL_MODE_ADDDATASTORE, + PZTOOL_MODE_DATASTORE, + PZTOOL_MODE_SEEN, + PZTOOL_MODE_PENDINGEXP, + PZTOOL_MODE_PENDINGIMFILE, + PZTOOL_MODE_COPYDONE, + PZTOOL_MODE_COPIED, + PZTOOL_MODE_UPDATECOPIED, + PZTOOL_MODE_REVERTCOPIED, + PZTOOL_MODE_CLEARCOMMONFAULTS, + PZTOOL_MODE_ADVANCE +} pztoolMode; + +pxConfig *pztoolConfig(pxConfig *config, int argc, char **argv); + +#endif // PZTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/pztoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/pztoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/pztoolConfig.c (revision 22235) @@ -0,0 +1,177 @@ +/* + * pztoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pztool.h" + +pxConfig *pztoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + // -adddatastore + psMetadata *adddatastoreArgs = psMetadataAlloc(); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-uri", 0, "define storage uri", NULL); + + // -datastore + psMetadata *datastoreArgs = psMetadataAlloc(); + psMetadataAddBool(datastoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -seen + psMetadata *seenArgs = psMetadataAlloc(); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddBool(seenArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-desc", 0, "sort ouput in descending format", false); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-desc", 0, "sort ouput in descending format", false); + psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -copydone + psMetadata *copydoneArgs = psMetadataAlloc(); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID (required)", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID (required)", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID (required)", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-class", 0, "define class", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-uri", 0, "define storage uri", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-workdir", 0, "define the \"default\" workdir for this exposure", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-hostname", 0, "define the host that copied the image", NULL); + psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddBool(copydoneArgs, PS_LIST_TAIL, "-row_lock", 0, "lock pzDownImfile rows while advancing an exposure", false); + + // -copied + psMetadata *copiedArgs = psMetadataAlloc(); + psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); + psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-class", 0, "define class", NULL); + psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id", NULL); + psMetadataAddBool(copiedArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddU64(copiedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(copiedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updatecopied + psMetadata *updatecopiedArgs = psMetadataAlloc(); + psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID (required)", NULL); + psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID (required)", NULL); + psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID (required)", NULL); + psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-class", 0, "define class", NULL); + psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id", NULL); + psMetadataAddS16(updatecopiedArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -revertcopied + psMetadata *revertcopiedArgs = psMetadataAlloc(); + psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID (required)", NULL); + psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID (required)", NULL); + psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID (required)", NULL); + psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-class", 0, "define class", NULL); + psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id", NULL); + psMetadataAddS16(revertcopiedArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -clearcommonfaults + psMetadata *clearcommonfaultsArgs = psMetadataAlloc(); + + // -advance + psMetadata *advanceArgs = psMetadataAlloc(); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-workdir", 0, "define the \"default\" workdir for this exposure", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-adddatastore", "", PZTOOL_MODE_ADDDATASTORE, adddatastoreArgs); + PXOPT_ADD_MODE("-datastore", "", PZTOOL_MODE_DATASTORE, datastoreArgs); + PXOPT_ADD_MODE("-seen", "", PZTOOL_MODE_SEEN, seenArgs); + PXOPT_ADD_MODE("-pendingexp", "", PZTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-pendingimfile", "", PZTOOL_MODE_PENDINGIMFILE,pendingimfileArgs); + PXOPT_ADD_MODE("-copydone", "", PZTOOL_MODE_COPYDONE, copydoneArgs); + PXOPT_ADD_MODE("-copied", "", PZTOOL_MODE_COPIED, copiedArgs); + PXOPT_ADD_MODE("-updatecopied", "", PZTOOL_MODE_UPDATECOPIED,updatecopiedArgs); + PXOPT_ADD_MODE("-revertcopied", "", PZTOOL_MODE_REVERTCOPIED,revertcopiedArgs); + PXOPT_ADD_MODE("-clearcommonfaults","", PZTOOL_MODE_CLEARCOMMONFAULTS,clearcommonfaultsArgs); + PXOPT_ADD_MODE("-advance", "", PZTOOL_MODE_ADVANCE, advanceArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/regtool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/regtool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/regtool.c (revision 22235) @@ -0,0 +1,1107 @@ +/* + * regtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "regtool.h" +#include "chiptool.h" + +// imfile +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool revertprocessedimfileMode(pxConfig *config); +static bool updateprocessedimfileMode(pxConfig *config); +// exp +static bool pendingexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool cleardupexpMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = regtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + // imfile + MODECASE(REGTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); + MODECASE(REGTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(REGTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); + MODECASE(REGTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + // exp + MODECASE(REGTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(REGTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(REGTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(REGTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(REGTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(REGTOOL_MODE_CLEARDUPEXP, cleardupexpMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // select newImfiles that: + // exp_id is in newExp + // don't have their exp_id in rawExp + // XXX having the same exp_id in newExp and raw*Exp is probably an error + // that should be checked for + + psString query = pxDataGet("regtool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regPendingImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(tmp_class_id, config->args, "-tmp_class_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + // optional + PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + PXOPT_LOOKUP_STR(obs_mode, config->args, "-obs_mode", false, false); + PXOPT_LOOKUP_STR(obs_group, config->args, "-obs_group", false, false); + + PXOPT_LOOKUP_F32(airmass, config->args, "-airmass", false, false); + PXOPT_LOOKUP_F64(ra, config->args, "-ra", false, false); + PXOPT_LOOKUP_F64(decl, config->args, "-decl", false, false); + PXOPT_LOOKUP_F32(exp_time, config->args, "-exp_time", false, false); + PXOPT_LOOKUP_F32(sat_pixel_frac, config->args, "-sat_pixel_frac", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(alt, config->args, "-alt", false, false); + PXOPT_LOOKUP_F64(az, config->args, "-az", false, false); + PXOPT_LOOKUP_F32(ccd_temp, config->args, "-ccd_temp", false, false); + PXOPT_LOOKUP_F64(posang, config->args, "-posang", false, false); + PXOPT_LOOKUP_F32(m1_x, config->args, "-m1_x", false, false); + PXOPT_LOOKUP_F32(m1_y, config->args, "-m1_y", false, false); + PXOPT_LOOKUP_F32(m1_z, config->args, "-m1_z", false, false); + PXOPT_LOOKUP_F32(m1_tip, config->args, "-m1_tip", false, false); + PXOPT_LOOKUP_F32(m1_tilt, config->args, "-m1_tilt", false, false); + PXOPT_LOOKUP_F32(m2_x, config->args, "-m2_x", false, false); + PXOPT_LOOKUP_F32(m2_y, config->args, "-m2_y", false, false); + PXOPT_LOOKUP_F32(m2_z, config->args, "-m2_z", false, false); + PXOPT_LOOKUP_F32(m2_tip, config->args, "-m2_tip", false, false); + PXOPT_LOOKUP_F32(m2_tilt, config->args, "-m2_tilt", false, false); + + PXOPT_LOOKUP_F32(env_temp, config->args, "-env_temperature", false, false); + PXOPT_LOOKUP_F32(env_humid, config->args, "-env_humidity", false, false); + PXOPT_LOOKUP_F32(env_wind, config->args, "-env_wind_speed", false, false); + PXOPT_LOOKUP_F32(env_dir, config->args, "-env_wind_dir", false, false); + PXOPT_LOOKUP_F32(teltemp_m1, config->args, "-teltemp_m1", false, false); + PXOPT_LOOKUP_F32(teltemp_m1cell, config->args, "-teltemp_m1cell", false, false); + PXOPT_LOOKUP_F32(teltemp_m2, config->args, "-teltemp_m2", false, false); + PXOPT_LOOKUP_F32(teltemp_spider, config->args, "-teltemp_spider", false, false); + PXOPT_LOOKUP_F32(teltemp_truss, config->args, "-teltemp_truss", false, false); + PXOPT_LOOKUP_F32(teltemp_extra, config->args, "-teltemp_extra", false, false); + PXOPT_LOOKUP_F32(pon_time, config->args, "-pon_time", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(object, config->args, "-object", false, false); + PXOPT_LOOKUP_F32(sun_angle, config->args, "-sun_angle", false, false); + PXOPT_LOOKUP_F32(sun_alt, config->args, "-sun_alt", false, false); + PXOPT_LOOKUP_F32(moon_angle, config->args, "-moon_angle", false, false); + PXOPT_LOOKUP_F32(moon_alt, config->args, "-moon_alt", false, false); + PXOPT_LOOKUP_F32(moon_phase, config->args, "-moon_phase", false, false); + PXOPT_LOOKUP_BOOL(ignored, config->args, "-ignore", false); + PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!rawImfileInsert( + config->dbh, + exp_id, + exp_name, + camera, + telescope, + dateobs, + tmp_class_id, + class_id, + uri, + exp_type, + filelevel, + filter, + comment, + obs_mode, + obs_group, + airmass, + ra, + decl, + exp_time, + sat_pixel_frac, + bg, + bg_stdev, + bg_mean_stdev, + alt, + az, + ccd_temp, + posang, + m1_x, + m1_y, + m1_z, + m1_tip, + m1_tilt, + m2_x, + m2_y, + m2_z, + m2_tip, + m2_tilt, + env_temp, + env_humid, + env_wind, + env_dir, + teltemp_m1, + teltemp_m1cell, + teltemp_m2, + teltemp_spider, + teltemp_truss, + teltemp_extra, + pon_time, + user_1, + user_2, + user_3, + user_4, + user_5, + object, + sun_angle, + sun_alt, + moon_angle, + moon_alt, + moon_phase, + ignored, + hostname, + code, + NULL + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND rawImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tmp_class_id", "tmp_class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("regtool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool updateprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + if (!pxSetFaultCode(config->dbh, "rawImfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // return only exps that: + // are not in rawExp + // have ALL of their imfiles in rawImfile (by count) + // and have no associated imfiles left in newImfile + psString query = pxDataGet("regtool_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + // skip past the "hook" comment + psStringPrepend(&limitString, "\n"); + } + + // 1st arg: where hook, 2nd arg: limit hook + if (!p_psDBRunQuery(config->dbh, query, "", limitString ? limitString : "")) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + return false; + } + psFree(limitString); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regPendingExp", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // make sure that the exp_id(s) are ready to be updated based on: + // exp_id is not in rawExp + // exp_id is not in newImfile + // that the correct count of imfiles is in rawImfile + + // required + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(exp_tag, config->args, "-exp_tag", true, false); + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", true, false); + + // optional + PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false); + PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + PXOPT_LOOKUP_STR(obs_mode, config->args, "-obs_mode", false, false); + PXOPT_LOOKUP_STR(obs_group, config->args, "-obs_group", false, false); + PXOPT_LOOKUP_F32(airmass, config->args, "-airmass", false, false); + PXOPT_LOOKUP_F64(ra, config->args, "-ra", false, false); + PXOPT_LOOKUP_F64(decl, config->args, "-decl", false, false); + PXOPT_LOOKUP_F32(exp_time, config->args, "-exp_time", false, false); + PXOPT_LOOKUP_F32(sat_pixel_frac, config->args, "-sat_pixel_frac", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(alt, config->args, "-alt", false, false); + PXOPT_LOOKUP_F64(az, config->args, "-az", false, false); + PXOPT_LOOKUP_F32(ccd_temp, config->args, "-ccd_temp", false, false); + PXOPT_LOOKUP_F64(posang, config->args, "-posang", false, false); + PXOPT_LOOKUP_F32(m1_x, config->args, "-m1_x", false, false); + PXOPT_LOOKUP_F32(m1_y, config->args, "-m1_y", false, false); + PXOPT_LOOKUP_F32(m1_z, config->args, "-m1_z", false, false); + PXOPT_LOOKUP_F32(m1_tip, config->args, "-m1_tip", false, false); + PXOPT_LOOKUP_F32(m1_tilt, config->args, "-m1_tilt", false, false); + PXOPT_LOOKUP_F32(m2_x, config->args, "-m2_x", false, false); + PXOPT_LOOKUP_F32(m2_y, config->args, "-m2_y", false, false); + PXOPT_LOOKUP_F32(m2_z, config->args, "-m2_z", false, false); + PXOPT_LOOKUP_F32(m2_tip, config->args, "-m2_tip", false, false); + PXOPT_LOOKUP_F32(m2_tilt, config->args, "-m2_tilt", false, false); + + PXOPT_LOOKUP_F32(env_temp, config->args, "-env_temperature", false, false); + PXOPT_LOOKUP_F32(env_humid, config->args, "-env_humidity", false, false); + PXOPT_LOOKUP_F32(env_wind, config->args, "-env_wind_speed", false, false); + PXOPT_LOOKUP_F32(env_dir, config->args, "-env_wind_dir", false, false); + PXOPT_LOOKUP_F32(teltemp_m1, config->args, "-teltemp_m1", false, false); + PXOPT_LOOKUP_F32(teltemp_m1cell, config->args, "-teltemp_m1cell", false, false); + PXOPT_LOOKUP_F32(teltemp_m2, config->args, "-teltemp_m2", false, false); + PXOPT_LOOKUP_F32(teltemp_spider, config->args, "-teltemp_spider", false, false); + PXOPT_LOOKUP_F32(teltemp_truss, config->args, "-teltemp_truss", false, false); + PXOPT_LOOKUP_F32(teltemp_extra, config->args, "-teltemp_extra", false, false); + PXOPT_LOOKUP_F32(pon_time, config->args, "-pon_time", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(object, config->args, "-object", false, false); + PXOPT_LOOKUP_F32(sun_angle, config->args, "-sun_angle", false, false); + PXOPT_LOOKUP_F32(sun_alt, config->args, "-sun_alt", false, false); + PXOPT_LOOKUP_F32(moon_angle, config->args, "-moon_angle", false, false); + PXOPT_LOOKUP_F32(moon_alt, config->args, "-moon_alt", false, false); + PXOPT_LOOKUP_F32(moon_phase, config->args, "-moon_phase", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + + // default + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + psString query = pxDataGet("regtool_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = NULL; + { + // build a query to search by exp_id + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS64(where, PS_LIST_TAIL, "newExp.exp_id", 0, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + psFree(query); + return false; + } + + whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + if (whereClause) { + // skip past comment "hook" + psStringPrepend(&whereClause, "\n AND "); + } + } + + // 1st arg: where hook, 2nd arg: limit hook + if (!p_psDBRunQuery(config->dbh, query, whereClause ? whereClause : "", "")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "no pending newExp rows found"); + psFree(output); + return false; + } + // sanity check that we only got one row + if (psArrayLength(output) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have gotten 1 row but %lu rows were returned", psArrayLength(output)); + psFree(output); + return NULL; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // insert the exp into rawExp + psMetadata *row = output->data[0]; + // convert metadata into a newExp object + psMetadataConfigPrint(stdout, row); + newExpRow *newExp = newExpObjectFromMetadata(row); + psFree(output); + if (!newExp) { + psError(PS_ERR_UNKNOWN, false, "this should not happen"); + return false; + } + + // carry through these values + // new CLI options overwrite existing values + workdir = workdir ? workdir : newExp->workdir; + reduction = reduction ? reduction : newExp->reduction; + dvodb = dvodb ? dvodb : newExp->dvodb; + tess_id = tess_id ? tess_id : newExp->tess_id; + end_stage = end_stage ? end_stage : newExp->end_stage; + label = label ? label : newExp->label; + // don't free newExp until just before we return, or these refs will break + + if (!rawExpInsert(config->dbh, + exp_id, + exp_name, + camera, + telescope, + dateobs, + exp_tag, + exp_type, + filelevel, + workdir, + reduction, + dvodb, + tess_id, + end_stage, + filter, + comment, + obs_mode, + obs_group, + airmass, + ra, + decl, + exp_time, + sat_pixel_frac, + bg, + bg_stdev, + bg_mean_stdev, + alt, + az, + ccd_temp, + posang, + m1_x, + m1_y, + m1_z, + m1_tip, + m1_tilt, + m2_x, + m2_y, + m2_z, + m2_tip, + m2_tilt, + env_temp, + env_humid, + env_wind, + env_dir, + teltemp_m1, + teltemp_m1cell, + teltemp_m2, + teltemp_spider, + teltemp_truss, + teltemp_extra, + pon_time, + user_1, + user_2, + user_3, + user_4, + user_5, + object, + sun_angle, + sun_alt, + moon_angle, + moon_alt, + moon_phase, + hostname, + code, + NULL + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newExp); + return false; + } + + // set the state for the newExp to stop + if (!pxnewExpSetState(config, exp_id, "stop")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change newExp.state for exp_id: %"PRId64, exp_id); + psFree(newExp); + return false; + } + + // should we stop here and proceed on to the chip stage? + // NULL for end_stage means go as far as possible + if (end_stage && psStrcasestr(end_stage, "reg")) { + // then we are done here + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(newExp); + return true; + } + // else continue on... + + + // insert an entry into the chipPendingExp table + // this can only be run as the newExp's state has been set to stop + if (!pxchipQueueByExpTag(config, + exp_id, + workdir, + label, + reduction, + NULL, // expgroup + dvodb, + tess_id, + end_stage + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue chipPendingExp"); + psFree(newExp); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newExp); + return false; + } + + psFree(newExp); + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("regtool_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XX test this out; need to make this consistent with the list in regtoolConfig.c + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F32(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("regtool_revertprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(where); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + if (!pxSetFaultCode(config->dbh, "rawExp", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree(where); + return false; + } + psFree(where); + + return true; +} + + +static bool cleardupexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + +// create temp table of exp_ids to be removed +{ + psString query = pxDataGet("regtool_create_dup_table.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); +} + +// populate that table +// XXX note that this query currently doesn't not correctly handle the case +// where there is more than one duplicate +{ + psString query = pxDataGet("regtool_populate_dup_table.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); +} + + if (!p_psDBRunQuery(config->dbh, "DELETE FROM rawImfile USING rawImfile, duplicate WHERE duplicate.exp_id = rawImfile.exp_id")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, "DELETE FROM rawExp USING rawExp, duplicate WHERE duplicate.exp_id = rawExp.exp_id")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, "DELETE FROM newImfile USING newImfile, duplicate WHERE duplicate.exp_id = newImfile.exp_id")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, "DELETE FROM newExp USING newExp, duplicate WHERE duplicate.exp_id = newExp.exp_id")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/regtool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/regtool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/regtool.h (revision 22235) @@ -0,0 +1,42 @@ +/* + * regtool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef REGTOOL_H +#define REGTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + REGTOOL_MODE_NONE = 0x0, + REGTOOL_MODE_PENDINGIMFILE, + REGTOOL_MODE_ADDPROCESSEDIMFILE, + REGTOOL_MODE_PROCESSEDIMFILE, + REGTOOL_MODE_REVERTPROCESSEDIMFILE, + REGTOOL_MODE_UPDATEPROCESSEDIMFILE, + REGTOOL_MODE_PENDINGEXP, + REGTOOL_MODE_ADDPROCESSEDEXP, + REGTOOL_MODE_PROCESSEDEXP, + REGTOOL_MODE_REVERTPROCESSEDEXP, + REGTOOL_MODE_UPDATEPROCESSEDEXP, + REGTOOL_MODE_CLEARDUPEXP, +} regtoolMode; + +pxConfig *regtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // REGTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/regtoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/regtoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/regtoolConfig.c (revision 22235) @@ -0,0 +1,318 @@ +/* + * regtoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "regtool.h" + +#define ADD_OPT(TYPE,TARG,NAME,COMMENT,DEFAULT) psMetadataAdd##TYPE(TARG, PS_LIST_TAIL, NAME, 0, COMMENT, DEFAULT) + +pxConfig *regtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PXTOOLS_ERR_CONFIG, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + ADD_OPT(U64, pendingimfileArgs, "-limit", "limit result set to N items", 0); + ADD_OPT(Bool, pendingimfileArgs, "-simple", "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + ADD_OPT(S64, addprocessedimfileArgs, "-exp_id", "define exposure ID (required)", 0); + ADD_OPT(Str, addprocessedimfileArgs, "-exp_name", "define the exp_name (required)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-inst", "define the camera name (required)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-telescope", "define the telescope name (required)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-tmp_class_id", "define temp. class ID (required)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-class_id", "define class ID (required)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-uri", "define URI (required)", NULL); + + ADD_OPT(F32, addprocessedimfileArgs, "-longitude", "specify the observatory longitude (NOTE: not saved in db)", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-latitude", "specify the observatory latitude (NOTE: not saved in db)", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-elevation", "specify the elevation (NOTE: not saved in db)", NAN); + + ADD_OPT(Str, addprocessedimfileArgs, "-exp_type", "define exposure type", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-filelevel", "define filelevel", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-filter", "define filter ", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-comment", "define comment ", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-obs_mode", "define observing mode (data usage goal)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-obs_group", "define observing group (set of associated observations)", NULL); + + ADD_OPT(F32, addprocessedimfileArgs, "-airmass", "define airmass", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-ra", "define RA", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-decl", "define DEC", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-exp_time", "define exposure time", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-sat_pixel_frac", "define fraction of saturated pixels", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-bg", "define exposue background", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-bg_stdev", "define exposue background stdev", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-bg_mean_stdev", "define exposue background mean stdev", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-alt", "define altitute", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-az", "define azimuth", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-ccd_temp", "define ccd tempature", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-posang", "define rotator position angle", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m1_x", "define M1 X position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m1_y", "define M1 Y position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m1_z", "define M1 Z position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m1_tip", "define M1 TIP position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m1_tilt", "define M1 TILT position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m2_x", "define M2 X position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m2_y", "define M2 Y position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m2_z", "define M2 Z position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m2_tip", "define M2 TIP position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m2_tilt", "define M2 TILT position", NAN); + + ADD_OPT(F32, addprocessedimfileArgs, "-env_temperature","define Environmental Temperature", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-env_humidity", "define Environmental Humidity", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-env_wind_speed", "define Environmental Wind Speed", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-env_wind_dir", "define Environmental Wind Direction", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_m1", "define Telescope Temperature : M1", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_m1cell", "define Telescope Temperature : M1 Cell", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_m2", "define Telescope Temperature : M2", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_spider", "define Telescope Temperature : Spider", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_truss", "define Telescope Temperature : Truss", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_extra", "define Telescope Temperature : Extra", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-pon_time", "define time to last Power On", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-user_1", "define user statistic (1)", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-user_2", "define user statistic (2)", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-user_3", "define user statistic (3)", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-user_4", "define user statistic (4)", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-user_5", "define user statistic (5)", NAN); + ADD_OPT(Str, addprocessedimfileArgs, "-object", "define exposure object", NULL); + ADD_OPT(F32, addprocessedimfileArgs, "-sun_angle", "define angle to sun", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-sun_alt", "define sun altitude (neg = below horizon)", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-moon_angle", "define angle to moon", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-moon_alt", "define moon altitude (neg = below horizon)", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-moon_phase", "define moon phase (0.0 = new)", NAN); + ADD_OPT(Bool, addprocessedimfileArgs, "-ignore", "ignore this imfile?", false); + ADD_OPT(Time, addprocessedimfileArgs, "-dateobs", "define observation time", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-hostname", "define host name", NULL); + ADD_OPT(S16, addprocessedimfileArgs, "-code", "set fault code", 0); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + ADD_OPT(S64, processedimfileArgs, "-exp_id", "search by exposure ID", 0); + ADD_OPT(Str, processedimfileArgs, "-exp_name", "search by exposure name", NULL); + ADD_OPT(Str, processedimfileArgs, "-class_id", "search by class ID", NULL); + ADD_OPT(U64, processedimfileArgs, "-limit", "limit result set to N items", 0); + ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); + ADD_OPT(Bool, processedimfileArgs, "-simple", "use the simple output format", false); + + // -revertprocessedimfile + psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); + ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id", "search by exposure ID (required)", 0); + ADD_OPT(Str, revertprocessedimfileArgs, "-tmp_class_id", "searcy by temp. class ID", NULL); + ADD_OPT(Str, revertprocessedimfileArgs, "-class_id", "search by class ID", NULL); + ADD_OPT(S16, revertprocessedimfileArgs, "-code", "search by fault code", 0); + + // -updateprocessedimfile + psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); + ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id", "search by exposure ID", 0); + ADD_OPT(Str, updateprocessedimfileArgs, "-class_id", "search by class ID", NULL); + ADD_OPT(S16, updateprocessedimfileArgs, "-code", "set fault code (required)", INT16_MAX); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "exp_id to operate on (required)", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-exp_name", 0, "define the exp_name (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-inst", 0, "define the camera name (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-telescope", 0, "define the telescope name (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "define the external exposure tag name (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-filelevel", 0, "define the data partitioning level of this file (required)", NULL); + + ADD_OPT(F32, addprocessedexpArgs, "-longitude", "specify the observatory longitude (NOTE: not saved in db)", 0.0); + ADD_OPT(F32, addprocessedexpArgs, "-latitude", "specify the observatory latitude (NOTE: not saved in db)", 0.0); + + ADD_OPT(Time, addprocessedexpArgs, "-dateobs", "define observation time", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-exp_type", "define exposure type", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-workdir", "define the \"default\" workdir for this exposure", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-dvodb", "define the dvodb for the next processing step", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-tess_id", "define the tess_id for the next processing step", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-end_stage", "define the end goal processing step", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-reduction", "define the \"default\" reduction class for this exposure", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-filter", "define filter ", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-comment", "define comment ", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-obs_mode", "define observing mode (data usage goal)", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-obs_group", "define observing group (set of associated observations)", NULL); + + ADD_OPT(F32, addprocessedexpArgs, "-airmass", "define airmass", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-ra", "define RA", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-decl", "define DEC", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-exp_time", "define exposure time", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-sat_pixel_frac", "define fraction of saturated pixels", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-bg", "define exposue background", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-bg_stdev", "define exposue background stdev", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-bg_mean_stdev", "define exposue background mean stdev", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-alt", "define altitute", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-az", "define azimuth", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-ccd_temp", "define ccd tempature", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-posang", "define rotator position angle", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-m1_x", "define M1 X position", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-m1_y", "define M1 Y position", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-m1_z", "define M1 Z position", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-m1_tip", "define M1 TIP position", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-m1_tilt", "define M1 TILT position", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-m2_x", "define M2 X position", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-m2_y", "define M2 Y position", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-m2_z", "define M2 Z position", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-m2_tip", "define M2 TIP position", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-m2_tilt", "define M2 TILT position", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-env_temperature", "define Environmental Temperature", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-env_humidity", "define Environmental Humidity", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-env_wind_speed", "define Environmental Wind Speed", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-env_wind_dir", "define Environmental Wind Direction", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-teltemp_m1", "define Telescope Temperature : M1", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-teltemp_m1cell", "define Telescope Temperature : M1 Cell", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-teltemp_m2", "define Telescope Temperature : M2", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-teltemp_spider", "define Telescope Temperature : Spider", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-teltemp_truss", "define Telescope Temperature : Truss", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-teltemp_extra", "define Telescope Temperature : Extra", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-pon_time", "define time to last Power On", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-user_1", "define user statistic (1)", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-user_2", "define user statistic (2)", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-user_3", "define user statistic (3)", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-user_4", "define user statistic (4)", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-user_5", "define user statistic (5)", NAN); + ADD_OPT(Str, addprocessedexpArgs, "-object", "define exposure object", NULL); + ADD_OPT(F32, addprocessedexpArgs, "-sun_angle", "define angle to sun", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-sun_alt", "define sun altitude (neg = below horizon)", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-moon_angle", "define angle to moon", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-moon_alt", "define moon altitude (neg = below horizon)", NAN); + ADD_OPT(F32, addprocessedexpArgs, "-moon_phase", "define moon phase (0.0 = new)", NAN); + ADD_OPT(Str, addprocessedexpArgs, "-label", "define label for chip stage (non-detrend data only)", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-hostname", "define host name", NULL); + ADD_OPT(S16, addprocessedexpArgs, "-code", "set fault code", 0); + + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_min", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-az_min", 0, "search by min", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-az_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertprocessedexp + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID (required)", 0); + psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatedprocessedexp + psMetadata *updatedprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updatedprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddS16(updatedprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", INT16_MAX); + + // -cleardupexp + psMetadata *cleardupexpArgs = psMetadataAlloc(); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-pendingimfile", "", REGTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-addprocessedimfile", "", REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXOPT_ADD_MODE("-processedimfile", "", REGTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXOPT_ADD_MODE("-revertprocessedimfile", "", REGTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); + PXOPT_ADD_MODE("-updateprocessedimfile", "", REGTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); + PXOPT_ADD_MODE("-pendingexp", "", REGTOOL_MODE_PENDINGEXP,pendingexpArgs); + PXOPT_ADD_MODE("-addprocessedexp", "", REGTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "", REGTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "", REGTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "", REGTOOL_MODE_UPDATEPROCESSEDEXP, updatedprocessedexpArgs); + PXOPT_ADD_MODE("-cleardupexp", "", REGTOOL_MODE_CLEARDUPEXP, cleardupexpArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PXTOOLS_ERR_SYS, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/stacktool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/stacktool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/stacktool.c (revision 22235) @@ -0,0 +1,1181 @@ +/* + * stacktool.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "stacktool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputskyfileMode(pxConfig *config); +static bool inputskyfileMode(pxConfig *config); +static bool tosumMode(pxConfig *config); +static bool addsumskyfileMode(pxConfig *config); +static bool sumskyfileMode(pxConfig *config); +static bool revertsumskyfileMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupskyfileMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool updatesumskyfileMode(pxConfig *config); + +static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = stacktoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(STACKTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(STACKTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(STACKTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(STACKTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileMode); + MODECASE(STACKTOOL_MODE_INPUTSKYFILE, inputskyfileMode); + MODECASE(STACKTOOL_MODE_TOSUM, tosumMode); + MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileMode); + MODECASE(STACKTOOL_MODE_SUMSKYFILE, sumskyfileMode); + MODECASE(STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileMode); + MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); + MODECASE(STACKTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(STACKTOOL_MODE_UPDATESUMSKYFILE, updatesumskyfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + + psMetadata *where = psMetadataAlloc(); + psMetadata *having = psMetadataAlloc(); // HAVING clause + + // select based on properties of the raw exposures + PXOPT_COPY_STR(config->args, where, "-select_inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-select_telescope", "rawExp.telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-select_filter", "rawExp.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-select_uri", "rawExp.uri", "=="); + PXOPT_COPY_TIME(config->args, where, "-select_dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-select_dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_F32(config->args, where, "-select_airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-select_airmass_max", "rawExp.airmass", "<="); + PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<="); + PXOPT_COPY_F32(config->args, where, "-select_exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-select_exp_time_max", "rawExp.exp_time", "<="); + PXOPT_COPY_F32(config->args, where, "-select_ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-select_ccd_temp_max", "rawExp.ccd_temp", "<="); + PXOPT_COPY_F32(config->args, where, "-select_posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F32(config->args, where, "-select_posang_max", "rawExp.posang", "<="); + PXOPT_COPY_F32(config->args, where, "-select_solang_min", "rawExp.solang", ">="); + PXOPT_COPY_F32(config->args, where, "-select_solang_max", "rawExp.solang", "<="); + PXOPT_COPY_STR(config->args, where, "-select_exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-select_label", "warpRun.label", "=="); + + // these are used to build the HAVING restriction + PXOPT_COPY_S32(config->args, having, "-min_num", "num_warp", ">="); + + // other options applied outside of the WHERE + PXOPT_LOOKUP_S32(randomLimit, config->args, "-random", false, false); + PXOPT_LOOKUP_S32(min_new, config->args, "-min_new", false, false); + PXOPT_LOOKUP_F32(min_frac, config->args, "-min_frac", false, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString select = pxDataGet("stacktool_definebyquery_part1.sql"); + if (!select) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&select, " AND %s", whereClause); + psFree(whereClause); + } + + psString groupby = pxDataGet("stacktool_definebyquery_part2.sql"); + if (!groupby) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + psStringAppend(&select, " %s", groupby); + psFree(groupby); + + // Restriction on aggregated quantities using HAVING + { + psString havingClause = NULL; // HAVING string + if (psListLength(having->list)) { + havingClause = psDBGenerateWhereConditionSQL(having, NULL); + } + + if (min_new > 0) { + if (havingClause) { + psStringAppend(&havingClause, " AND"); + } + psStringAppend(&havingClause, + " (num_warp - num_stack >= %d OR (num_warp >= %d AND num_stack IS NULL))", + min_new, min_new); + } + if (isfinite(min_frac)) { + if (havingClause) { + psStringAppend(&havingClause, " AND"); + } + // Avoiding division by zero + psStringAppend(&havingClause, " (num_warp >= %f * num_stack OR num_stack IS NULL)", + (double)min_frac); + } + if (havingClause) { + psStringAppend(&select, " HAVING %s", havingClause); + psFree(havingClause); + } + } + psFree(having); + + if (!p_psDBRunQuery(config->dbh, select)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(select); + psFree(where); + return false; + } + psFree(select); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + psFree(where); + return true; + } + + psString insert = NULL; // Insertion query + if (randomLimit > 0) { + insert = pxDataGet("stacktool_definebyquery_insert_random_part1.sql"); + } else { + insert = pxDataGet("stacktool_definebyquery_insert.sql"); + } + if (!insert) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&insert, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (randomLimit > 0) { + psString part2 = pxDataGet("stacktool_definebyquery_insert_random_part2.sql"); + if (!part2) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(insert); + return false; + } + psStringAppend(&insert, "%s", part2); + psFree(part2); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + // pull out the skycell_id, tess_id, filter + psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup skycell_id"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString tess_id = psMetadataLookupStr(&status, row, "tess_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup tess_id"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString filter = psMetadataLookupStr(&status, row, "filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup filter"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // create a new stackRun for this stack + stackRunRow *run = stackRunRowAlloc( + 0, // ID + "new", // state + workdir, + label, + reduction, + dvodb, + registered, + skycell_id, + tess_id, + filter); + + if (!stackRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // figure out the new stack_id + psS64 stack_id = psDBLastInsertID(config->dbh); + run->stack_id = stack_id; + + psArrayAdd(list, list->n, run); + psFree(run); + + // Create a suitable insertion query for this run + psString thisInsert = psStringCopy(insert); + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, stack_id); + psStringSubstitute(&thisInsert, idString, "@STACK_ID@"); + psFree(idString); + + // XXX this insert uses a select to generate the list of warp_ids for the stack, + // we have applied a set of criteria above (WHERE) to select the relevant warps + // this insert needs to use exactly the same restrictions (race condition is probably not critical) + // the insert below seems to only restrict matches to the skycell, tess, and filter + if ((randomLimit > 0 && !p_psDBRunQuery(config->dbh, thisInsert, skycell_id, filter, randomLimit)) || + (randomLimit <= 0 && !p_psDBRunQuery(config->dbh, thisInsert, skycell_id, filter))) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(thisInsert); + psFree(insert); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(thisInsert); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + if (!stackRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); + return true; +} + + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", true, false); + + // default + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + + // options + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + + // we have to support multipe exp_ids + psMetadataItem *warp_ids = psMetadataLookup(config->args, "-warp_id"); + if (!warp_ids) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-warp_id is required"); + return false; + } + + stackRunRow *run = stackRunRowAlloc( + 0, // ID + "new", // state + workdir, + label, + reduction, + dvodb, + registered, + skycell_id, + tess_id, + filter); + + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc stackRun object"); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!stackRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + + // get the assigned warp_id + run->stack_id = psDBLastInsertID(config->dbh); + + // insert the stackInputSkyfile rows + psListIterator *iter = psListIteratorAlloc(warp_ids->data.list, 0, false); + psMetadataItem *item = NULL; + while ((item = psListGetAndIncrement(iter))) { + // if the value is NULL this is probably the first pass through the + // loop and -warp_id was not specified at all + if (!item->data.V) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warp_id"); + return false; + } + if (!stackInputSkyfileInsert(config->dbh, run->stack_id, item->data.S64)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to insert stackInputSkyfile rows"); + return false; + } + } + psFree(iter); + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!stackRunPrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(run); + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (state) { + // set detRun.state to state + return setstackRunState(config, stack_id, state); + } + + return true; +} + + +static bool addinputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + + // XXX need to validate the warp_id here + // XXX instead of validiting it here we should just use forgein key + // constrants + if (!stackInputSkyfileInsert(config->dbh, + stack_id, + warp_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool inputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX require at least a stack id (add better search options) + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_inputskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "stackInputSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackInputSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool tosumMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_tosum.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addsumskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_stack, config->args, "-dtime_stack", false, false); + PXOPT_LOOKUP_F32(dtime_match_mean, config->args, "-dtime_match_mean", false, false); + PXOPT_LOOKUP_F32(dtime_match_stdev, config->args, "-dtime_match_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_initial, config->args, "-dtime_initial", false, false); + PXOPT_LOOKUP_F32(dtime_reject, config->args, "-dtime_reject", false, false); + PXOPT_LOOKUP_F32(dtime_final, config->args, "-dtime_final", false, false); + PXOPT_LOOKUP_F32(dtime_phot, config->args, "-dtime_phot", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_F32(match_mean, config->args, "-match_mean", false, false); + PXOPT_LOOKUP_F32(match_stdev, config->args, "-match_stdev", false, false); + PXOPT_LOOKUP_F32(match_rms, config->args, "-match_rms", false, false); + PXOPT_LOOKUP_F32(stamps_mean, config->args, "-stamps_mean", false, false); + PXOPT_LOOKUP_F32(stamps_stdev, config->args, "-stamps_stdev", false, false); + PXOPT_LOOKUP_S32(stamps_min, config->args, "-stamps_min", false, false); + PXOPT_LOOKUP_S32(reject_images, config->args, "-reject_images", false, false); + PXOPT_LOOKUP_F32(reject_pix_mean, config->args, "-reject_pix_mean", false, false); + PXOPT_LOOKUP_F32(reject_pix_stdev, config->args, "-reject_pix_stdev", false, false); + PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX need to validate the stack_id here + // XXX instead of validiting it here we should just use forgein key + // constrants + if (!stackSumSkyfileInsert(config->dbh, + stack_id, + uri, + path_base, + bg, + bg_stdev, + dtime_stack, + dtime_match_mean, + dtime_match_stdev, + dtime_initial, + dtime_reject, + dtime_final, + dtime_phot, + dtime_script, + match_mean, + match_stdev, + match_rms, + stamps_mean, + stamps_stdev, + stamps_min, + reject_images, + reject_pix_mean, + reject_pix_stdev, + sources, + hostname, + good_frac, + code + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!setstackRunState(config, stack_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change stackRun's state"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool sumskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_sumskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertsumskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "stackSumSkyfile.fault", "=="); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + int numStacks; // Number of stacks affected + + // Update state to 'new' + { + psString update = pxDataGet("stacktool_revertsumskyfile_update.sql"); + if (!update) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&update, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, update)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(update); + psFree(where); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(update); + + numStacks = psDBAffectedRows(config->dbh); + + if (numStacks < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + + // Delete product + { + psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); + if (!delete) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(delete); + + if (psDBAffectedRows(config->dbh) != numStacks) { + psError(PS_ERR_UNKNOWN, true, "Updated and deleted different number of entries!"); + psFree(where); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(where); + + return true; +} + + +static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); + return false; + } + + char *query = "UPDATE stackRun SET state = '%s' WHERE stack_id = %"PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, stack_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for stack_id %"PRId64, stack_id); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("stacktool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingcleanupskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (stack_id) { + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("stacktool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool updatesumskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + + if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/stacktool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/stacktool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/stacktool.h (revision 22235) @@ -0,0 +1,44 @@ +/* + * stacktool.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef STACKTOOL_H +#define STACKTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + STACKTOOL_MODE_NONE = 0x0, + STACKTOOL_MODE_DEFINEBYQUERY, + STACKTOOL_MODE_DEFINERUN, + STACKTOOL_MODE_UPDATERUN, + STACKTOOL_MODE_ADDINPUTSKYFILE, + STACKTOOL_MODE_INPUTSKYFILE, + STACKTOOL_MODE_TOSUM, + STACKTOOL_MODE_ADDSUMSKYFILE, + STACKTOOL_MODE_SUMSKYFILE, + STACKTOOL_MODE_REVERTSUMSKYFILE, + STACKTOOL_MODE_PENDINGCLEANUPRUN, + STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, + STACKTOOL_MODE_DONECLEANUP, + STACKTOOL_MODE_UPDATESUMSKYFILE, +} stacktoolMode; + +pxConfig *stacktoolConfig(pxConfig *config, int argc, char **argv); + +#endif // STACKTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/stacktoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/stacktoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/stacktoolConfig.c (revision 22235) @@ -0,0 +1,230 @@ +/* + * stacktoolConfig.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "stacktool.h" + +pxConfig *stacktoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, "search for exp_type", "object"); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", 0, "search for label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, "search for camera", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_telescope", 0, "search for telescope", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", 0, "search for filter", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_uri", 0, "search for uri", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max", 0, "define max airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, "define max solar angle", NAN); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_num", 0, "minimum number of inputs", 0); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_new", 0, "minimum number of new inputs", 0); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-min_frac", 0, "minumum fraction of new inputs", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-filter", 0, "define filter (required)", NULL); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-warp_id", PS_META_DUPLICATE_OK, "include this warp ID (multiple OK, required)", 0); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + + // -addinputskyfile + psMetadata *addinputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); + + // -inputskyfile + // XXX add additional search terms + psMetadata *inputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddU64(inputskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tosum + psMetadata *tosumArgs = psMetadataAlloc(); + psMetadataAddS64(tosumArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(tosumArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddS64(tosumArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddU64(tosumArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tosumArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsumskyfile + psMetadata *addsumskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(addsumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI of file", 0); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF64(addsumskyfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposue background", NAN); + psMetadataAddF64(addsumskyfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposue background mean stdev", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_stack", 0, "define elapsed processing time", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_match_mean", 0, "define mean matching time", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_match_stdev", 0, "define stdev matching time", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_initial", 0, "define initial stack time", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_reject", 0, "define rejection time", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_final", 0, "define final stack time", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_phot", 0, "define photometry time", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-match_mean", 0, "define mean matching deviation", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-match_stdev", 0, "define stdev matching deviation", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-match_rms", 0, "define mean rms of deviation", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-stamps_mean", 0, "define mean number of stamps", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-stamps_stdev", 0, "define stdev of number of stamps", NAN); + psMetadataAddS32(addsumskyfileArgs, PS_LIST_TAIL, "-stamps_min", 0, "define minimum number of stamps", 0); + psMetadataAddS32(addsumskyfileArgs, PS_LIST_TAIL, "-reject_images", 0, "number of images rejected", 0); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-reject_pix_mean", 0, "mean number of pixels rejected", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-reject_pix_stdev", 0, "stdev number of pixels rejected", NAN); + psMetadataAddS32(addsumskyfileArgs, PS_LIST_TAIL, "-sources", 0, "number of sources", 0); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); + psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -sumskyfile + psMetadata *sumskyfileArgs= psMetadataAlloc(); + psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); + psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertsumskyfile + psMetadata *revertsumskyfileArgs= psMetadataAlloc(); + psMetadataAddS64(revertsumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(revertsumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddS16(revertsumskyfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupskyfile + psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -updatesumskyfile + psMetadata *updatesumskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(updatesumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); + psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", 0); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "Define a new stackRun by searching for warp IDs", STACKTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "", STACKTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-updaterun", "", STACKTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-addinputskyfile", "", STACKTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs); + PXOPT_ADD_MODE("-inputskyfile", "", STACKTOOL_MODE_INPUTSKYFILE, inputskyfileArgs); + PXOPT_ADD_MODE("-tosum", "", STACKTOOL_MODE_TOSUM, tosumArgs); + PXOPT_ADD_MODE("-addsumskyfile", "", STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileArgs); + PXOPT_ADD_MODE("-sumskyfile", "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE, sumskyfileArgs); + PXOPT_ADD_MODE("-revertsumskyfile","", STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", STACKTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-updatesumskyfile", "update fault code for sumskyfile", STACKTOOL_MODE_UPDATESUMSKYFILE, updatesumskyfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/warptool.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/warptool.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/warptool.c (revision 22235) @@ -0,0 +1,1596 @@ +/* + * warptool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "warptool.h" +#include "pxwarp.h" + +static psS64 definerunMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool expMode(pxConfig *config); +static bool imfileMode(pxConfig *config); +static bool tooverlapMode(pxConfig *config); +static bool addoverlapMode(pxConfig *config); +static bool scmapMode(pxConfig *config); +static bool towarpedMode(pxConfig *config); +static bool addwarpedMode(pxConfig *config); +static bool warpedMode(pxConfig *config); +static bool revertwarpedMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupwarpMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool tocleanedskyfileMode(pxConfig *config); +static bool topurgedskyfileMode(pxConfig *config); +static bool tofullskyfileMode(pxConfig *config); +static bool updateskyfileMode(pxConfig *config); + +static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); +static bool isValidMode(pxConfig *config, const char *mode); +bool warpCompletedRuns(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = warptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(WARPTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(WARPTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(WARPTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(WARPTOOL_MODE_EXP, expMode); + MODECASE(WARPTOOL_MODE_IMFILE, imfileMode); + MODECASE(WARPTOOL_MODE_TOOVERLAP, tooverlapMode); + MODECASE(WARPTOOL_MODE_ADDOVERLAP, addoverlapMode); + MODECASE(WARPTOOL_MODE_SCMAP, scmapMode); + MODECASE(WARPTOOL_MODE_TOWARPED, towarpedMode); + MODECASE(WARPTOOL_MODE_ADDWARPED, addwarpedMode); + MODECASE(WARPTOOL_MODE_WARPED, warpedMode); + MODECASE(WARPTOOL_MODE_REVERTWARPED, revertwarpedMode); + MODECASE(WARPTOOL_MODE_BLOCK, blockMode); + MODECASE(WARPTOOL_MODE_MASKED, maskedMode); + MODECASE(WARPTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(WARPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupwarpMode); + MODECASE(WARPTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); + MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); + MODECASE(WARPTOOL_MODE_UPDATESKYFILE, updateskyfileMode); + + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static psS64 definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(fake_id, config->args, "-fake_id", true, false); // required + PXOPT_LOOKUP_STR(mode, config->args, "-mode", true, false); // required + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalid mode"); + return false; + } + + warpRunRow *warpRun = warpRunRowAlloc( + 0, // ID + fake_id, + mode, + "new", // state + workdir, + "dirty", // workdir_state + label, + dvodb, + tess_id, + end_stage, + registered, + magiced + ); + if (!warpRun) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc warpRun object"); + return true; + } + if (!warpRunInsertObject(config->dbh, warpRun)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpRun); + return true; + } + + // get the assigned warp_id + psS64 warp_id = psDBLastInsertID(config->dbh); + warpRun->warp_id = warp_id; + + if (!warpRunPrintObject(stdout, warpRun, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(warpRun); + return false; + } + + psFree(warpRun); + + return warp_id; +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "newExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(mode, config->args, "-set_mode", true, false); // required + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalid mode"); + return false; + } + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("warptool_definebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + // loop over our list of fakeRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + fakeRunRow *row = fakeRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fakeRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxwarpQueueByFakeID(config, + row->fake_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + dvodb ? dvodb : row->dvodb, + tess_id ? tess_id : row->tess_id, + end_stage ? end_stage : row->end_stage)) + { + psError(PS_ERR_UNKNOWN, false, "failed to trying to queue fake_id: %" PRId64, row->fake_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F32(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + where = NULL; + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters (-state or -label) are required"); + psFree(where); + return false; + } + + if (state) { + // set warpRun.state to state + if (!pxwarpRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set chipRun.label to label + if (!pxwarpRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool expMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool imfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_imfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpInputImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool tooverlapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_tooverlap.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addoverlapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(mapfile, config->args, "-mapfile", false, false); + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (code == 0) { + if (!parseAndInsertSkyCellMap(config, mapfile)) { + psError(PS_ERR_UNKNOWN, false, "failed to inject mapfile: %s into the database", mapfile); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } else { + warpSkyCellMapInsert(config->dbh, + warp_id, + NULL, // skycell_id + NULL, // tess_id + NULL, // class_id + code // fault + ); + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile) +{ + unsigned int nFail = 0; + psMetadata *skycells = psMetadataConfigRead(NULL, &nFail, mapfile, false); + if (!skycells) { + psError(PS_ERR_UNKNOWN, false, "failed to parse mapfile: %s", mapfile); + return false; + } + if (nFail) { + psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing mapfile: %s", nFail, mapfile); + psFree(skycells); + return false; + } + + psMetadataItem *item = NULL; + psMetadataIterator *iter = psMetadataIteratorAlloc(skycells, 0, NULL); + while ((item = psMetadataGetAndIncrement(iter))) { + if (item->type != PS_DATA_METADATA) { + psError(PS_ERR_UNKNOWN, false, "mapfile: %s is in the wrong format", mapfile); + psFree(iter); + psFree(skycells); + return false; + } + + psMetadata *sc = item->data.md; + // this conversion isn't strictly nessicary but it's an easy way of + // validating the format + warpSkyCellMapRow *row = warpSkyCellMapObjectFromMetadata(sc); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert mapfile: %s metdata entry into a warpSkyCellMap object", mapfile); + psFree(iter); + psFree(skycells); + return false; + } + + if (!warpSkyCellMapInsertObject(config->dbh, row)) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(row); + psFree(iter); + psFree(skycells); + return false; + } + + psFree(row); + } + psFree(iter); + psFree(skycells); + + return true; +} + + +static bool scmapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_scmap.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "warpSkyCellMap"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpSkyCellMap", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool towarpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_towarped.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addwarpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_warp, config->args, "-dtime_warp", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_S32(xmin, config->args, "-xmin", false, false); + PXOPT_LOOKUP_S32(xmax, config->args, "-xmax", false, false); + PXOPT_LOOKUP_S32(ymin, config->args, "-ymin", false, false); + PXOPT_LOOKUP_S32(ymax, config->args, "-ymax", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // we don't want to insert the last skyfile in a run but then not mark the + // run as 'stop' + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX need to validate that this coresponds to an warpInputImfile + if (!warpSkyfileInsert(config->dbh, + warp_id, + skycell_id, + tess_id, + uri, + path_base, + "full", // data_state + bg, + bg_stdev, + dtime_warp, + dtime_script, + hostname, + good_frac, + xmin, + xmax, + ymin, + ymax, + !accept, + code + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!warpCompletedRuns(config)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +bool warpCompletedRuns(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX this SQL has not been broken out to into seperate files as the MYSQL + // < 5 & MYSQL 5 versions need to be kept in sync + +#undef MYSQL5 +#if MYSQL5 + // XXX at MySQL 4.1.21 (probably all of 4.1.x) chokes and dies on this + // statement as it thinks it is trying to select from the table being + // updated. The 4.1 manual says that nested sub-queries are explicited + // allowed to do this with update statements as a temporary table is + // created so that you are not actually selecting from the table you are + // modifying. + char *query = + "UPDATE warpRun\n" + " SET warpRun.state = 'stop'\n" + " WHERE\n" + " warpRun.warp_id =\n" + " (SELECT DISTINCT\n" + " warp_id\n" + " FROM\n" + " (SELECT DISTINCT\n" + " warpRun.warp_id,\n" + " warpSkyCellMap.warp_id as foo,\n" + " warpSkyfile.warp_id as bar\n" + " FROM warpRun\n" + " JOIN warpSkyCellMap\n" + " USING(warp_id)\n" + " LEFT JOIN warpSkyfile\n" + " USING(warp_id, skycell_id, tess_id)\n" + " WHERE\n" + " warpRun.state = 'new'\n" + " GROUP BY\n" + " warpRun.warp_id\n" + " HAVING\n" + " COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n" + " ) as Foo\n" + " )\n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +#else // if MYSQL5 +{ + char *query = + "CREATE TEMPORARY TABLE finished\n" + " (warp_id INT, PRIMARY KEY(warp_id)) ENGINE=MEMORY\n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +} + +{ + char *query = + "INSERT INTO finished\n" + " SELECT\n" + " warp_id\n" + " FROM\n" + " (SELECT DISTINCT\n" + " warpRun.warp_id,\n" + " warpSkyCellMap.warp_id as foo,\n" + " warpSkyfile.warp_id as bar\n" + " FROM warpRun\n" + " JOIN warpSkyCellMap\n" + " USING(warp_id)\n" + " LEFT JOIN warpSkyfile\n" + " USING(warp_id, skycell_id)\n" + " WHERE\n" + " warpRun.state = 'new'\n" + " GROUP BY\n" + " warpRun.warp_id\n" + " HAVING\n" + " COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n" + " ) as Foo \n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +} + +{ + char *query = + "UPDATE warpRun\n" + " SET warpRun.state = 'full'\n" + " WHERE\n" + " warpRun.warp_id =\n" + " (SELECT DISTINCT\n" + " warp_id\n" + " FROM finished\n" + " )\n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +} +#endif // if MYSQL5 + + return true; +} + +static bool warpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_warped.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertwarpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + + PXOPT_COPY_F32(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); + PXOPT_COPY_S16(config->args, where, "-code", "warpSkyfile.fault", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("warptool_revertwarped.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!warpMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM warpMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warpool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM warpMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("warptool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (warp_id) { + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("warptool_pendingcleanupskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpPendingCleanupWarp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("warptool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool isValidMode(pxConfig *config, const char *mode) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(mode, false); + + // check that state is a valid string value + if (!( + (strncmp(mode, "warp", 5) == 0) + || (strncmp(mode, "diff", 5) == 0) + || (strncmp(mode, "stack", 6) == 0) + || (strncmp(mode, "magic", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid warpRun mode: %s", mode); + return false; + } + + return true; +} + +// update warpSkyfile.data_state to given value. +// afterwards, if all skfyiles in the run have the new state, update the state for the run as well +// shared code for the modes -tocleanedskyfile -tofullskyfile -topurgedskyfile + +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // warp_id, skycell_id are required + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + psString query = pxDataGet("warptool_change_skyfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // note only updates if warpRun.state = run_state + if (!p_psDBRunQuery(config->dbh, query, data_state, warp_id, skycell_id, run_state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + query = pxDataGet("warptool_change_run_state.sql"); + if (!p_psDBRunQuery(config->dbh, query, data_state, warp_id, data_state)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool tocleanedskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "cleaned", "goto_cleaned"); +} +static bool tofullskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "full", "update"); +} +static bool topurgedskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "purged", "goto_purged"); +} + +static bool updateskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // warp_id, skycell_id, code are required + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + psString query = pxDataGet("warptool_updateskyfile.sql"); + + if (!p_psDBRunQuery(config->dbh, query, code, warp_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + return true; +} Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/warptool.h =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/warptool.h (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/warptool.h (revision 22235) @@ -0,0 +1,55 @@ +/* + * warptool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef WARPTOOL_H +#define WARPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + WARPTOOL_MODE_NONE = 0x0, + WARPTOOL_MODE_DEFINEBYQUERY, + WARPTOOL_MODE_DEFINERUN, + WARPTOOL_MODE_RUNONE, + WARPTOOL_MODE_UPDATERUN, + WARPTOOL_MODE_ADDINPUTEXP, + WARPTOOL_MODE_EXP, + WARPTOOL_MODE_IMFILE, + WARPTOOL_MODE_TOOVERLAP, + WARPTOOL_MODE_ADDOVERLAP, + WARPTOOL_MODE_SCMAP, + WARPTOOL_MODE_TOWARPED, + WARPTOOL_MODE_ADDWARPED, + WARPTOOL_MODE_WARPED, + WARPTOOL_MODE_REVERTWARPED, + WARPTOOL_MODE_BLOCK, + WARPTOOL_MODE_MASKED, + WARPTOOL_MODE_UNBLOCK, + WARPTOOL_MODE_PENDINGCLEANUPRUN, + WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, + WARPTOOL_MODE_DONECLEANUP, + WARPTOOL_MODE_TOCLEANEDSKYFILE, + WARPTOOL_MODE_TOPURGEDSKYFILE, + WARPTOOL_MODE_TOFULLSKYFILE, + WARPTOOL_MODE_UPDATESKYFILE, +} warptoolMode; + +pxConfig *warptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // WARPTOOL_H Index: /branches/pap_branches/pap_branch_20081109/ippTools/src/warptoolConfig.c =================================================================== --- /branches/pap_branches/pap_branch_20081109/ippTools/src/warptoolConfig.c (revision 22235) +++ /branches/pap_branches/pap_branch_20081109/ippTools/src/warptoolConfig.c (revision 22235) @@ -0,0 +1,407 @@ +/* + * warptoolConfig.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "warptool.h" + +pxConfig *warptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "search on fakeRun label", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_mode", 0, "define mode (warp, diff, stack, magic)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tess ID", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-magiced", 0, "has this exposure been magiced", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-fake_id", 0, "define camtool ID (required)", 0); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-mode", 0, "define mode (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tess_id", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-end_stage", 0, "define end stage", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-magiced", 0, "has this exposure been magiced", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + // XXX need to allow multiple fake_ids + // XXX need to allow multiple fake_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "set label", NULL); + +#if 0 + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); +#endif + + // -exp + psMetadata *expArgs = psMetadataAlloc(); + psMetadataAddS64(expArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddS64(expArgs, PS_LIST_TAIL, "-fake_id", 0, "search by camtool ID", 0); + psMetadataAddU64(expArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(expArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -imfile + psMetadata *imfileArgs = psMetadataAlloc(); + psMetadataAddS64(imfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddS64(imfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by camtool ID", 0); + psMetadataAddU64(imfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(imfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tooverlap + psMetadata *tooverlapArgs = psMetadataAlloc(); + psMetadataAddS64(tooverlapArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(tooverlapArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddU64(tooverlapArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tooverlapArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addoverlap + psMetadata *addoverlapArgs = psMetadataAlloc(); + psMetadataAddStr(addoverlapArgs, PS_LIST_TAIL, "-mapfile", 0, "path to skycell <-> imfile mapping file", NULL); + psMetadataAddS64(addoverlapArgs, PS_LIST_TAIL, "-warp_id", 0, "set warp ID", 0); + psMetadataAddS16(addoverlapArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -scmap + psMetadata *scmapArgs = psMetadataAlloc(); + psMetadataAddS64(scmapArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(scmapArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(scmapArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddStr(scmapArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddU64(scmapArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(scmapArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -towarped + psMetadata *towarpedArgs = psMetadataAlloc(); + psMetadataAddS64(towarpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(towarpedArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddU64(towarpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(towarpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addwarped + psMetadata *addwarpedArgs = psMetadataAlloc(); + psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-warp_id", 0, "define warptool ID (required)", 0); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-uri", 0, "define URI of file", 0); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF64(addwarpedArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addwarpedArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-dtime_warp", 0, "define elapsed processing time", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-xmin", 0, "define minimum x value", INT_MAX); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-xmax", 0, "define maximum x value", -INT_MAX); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-ymin", 0, "define minimum y value", INT_MAX); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-ymax", 0, "define maximum y value", -INT_MAX); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); + psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-accept", 0, "define if this skycell should be accepted", false); + psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -warped + psMetadata *warpedArgs = psMetadataAlloc(); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure tag", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure tag", 0); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "define phase 3 version of exposure tag", 0); + psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertwarped + // XXX need to allow multiple fake_ids + // XXX need to allow multiple fake_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *revertwarpedArgs = psMetadataAlloc(); + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, "searcy by tessellation ID", NULL); + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake_id", 0); + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(revertwarpedArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(revertwarpedArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + + psMetadataAddS16(revertwarpedArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + psMetadataAddBool(revertwarpedArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupskyfile + psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -tocleanedskyfile + psMetadata *tocleanedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(tocleanedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -topurgedskyfile + psMetadata *topurgedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(topurgedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -tofullskyfile + psMetadata *tofullskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -toupdateskyfile + psMetadata *updateskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-code", 0, "new fault code", 0); + + psFree(now); + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "", WARPTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "", WARPTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-updaterun", "", WARPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-exp", "", WARPTOOL_MODE_EXP, expArgs); + PXOPT_ADD_MODE("-imfile", "", WARPTOOL_MODE_IMFILE, imfileArgs); + PXOPT_ADD_MODE("-tooverlap", "", WARPTOOL_MODE_TOOVERLAP, tooverlapArgs); + PXOPT_ADD_MODE("-addoverlap", "", WARPTOOL_MODE_ADDOVERLAP, addoverlapArgs); + PXOPT_ADD_MODE("-scmap", "", WARPTOOL_MODE_SCMAP, scmapArgs); + PXOPT_ADD_MODE("-towarped", "", WARPTOOL_MODE_TOWARPED, towarpedArgs); + PXOPT_ADD_MODE("-addwarped", "", WARPTOOL_MODE_ADDWARPED, addwarpedArgs); + PXOPT_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); + PXOPT_ADD_MODE("-revertwarped", "", WARPTOOL_MODE_REVERTWARPED, revertwarpedArgs); + PXOPT_ADD_MODE("-block", "set a label block", WARPTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked lables", WARPTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", WARPTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", WARPTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs); + PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); + PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); + PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +}