Index: trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c =================================================================== --- trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (revision 25323) +++ trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (revision 25324) @@ -292,4 +292,19 @@ CONCAT_FITS = argv[i][10] == 'n' || argv[i][10] == '0' || argv[i][10] == 'f'; + +/* Set the e/ADU for the noise gate that cells must pass */ + } else if(strncmp(argv[i], "EADU=", 5) == 0) { /* EADU=e */ + if(sscanf(argv[i]+5, "%lf", &MIN_EADU) != 1) { + fprintf(stderr, "\rerror: cannot get e/ADU size from `%s'\n", argv[i]); + exit(EXIT_FAILURE); + } + + + /* Set the read noise for the noise gate that cells must pass */ + } else if(strncmp(argv[i], "RN=", 3) == 0) { /* EADU=e */ + if(sscanf(argv[i]+3, "%d", &MAX_READ_NOISE) != 1) { + fprintf(stderr, "\rerror: cannot get read noise size from `%s'\n", argv[i]); + exit(EXIT_FAILURE); + } /* Quiet? */ Index: trunk/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 =================================================================== --- trunk/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 (revision 25323) +++ trunk/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 (revision 25324) @@ -303,4 +303,15 @@ Retire a blasted burn after N seconds + EADU=x + Set the minimum e/ADU for acceptable cell sky noise + (default 0.3). Should not normally be changed! + RN=x + Set the maximum read noise for acceptable cell sky noise + (default 20). Should not normally be changed! + + Any cell whose noise variance in the sky is greater than + sky/EADU+RN*RN is rejected out of hand, and skipped for + further processing. + quiet={t|f} Quiet? Index: trunk/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- trunk/ippScripts/scripts/ipp_cleanup.pl (revision 25323) +++ trunk/ippScripts/scripts/ipp_cleanup.pl (revision 25324) @@ -154,9 +154,9 @@ addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); # clean? + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); # clean? addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); #clean? addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); } @@ -253,5 +253,5 @@ my @files = (); # delete the temporary image datafiles - addFilename (\@files, "PSASTRO.OUTPUT", $path_base); +# addFilename (\@files, "PSASTRO.OUTPUT", $path_base); if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode @@ -349,5 +349,5 @@ addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); - addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); +# addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); if ($mode eq "goto_purged") { @@ -561,4 +561,5 @@ addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id); addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id); addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id); Index: trunk/ippTasks/Makefile.am =================================================================== --- trunk/ippTasks/Makefile.am (revision 25323) +++ trunk/ippTasks/Makefile.am (revision 25324) @@ -26,5 +26,6 @@ pstamp.pro \ receive.pro \ - publish.pro + publish.pro \ + science.cleanup.pro other_files = \ Index: trunk/ippTasks/detrend.cleanup.pro =================================================================== --- trunk/ippTasks/detrend.cleanup.pro (revision 25323) +++ trunk/ippTasks/detrend.cleanup.pro (revision 25324) @@ -234,5 +234,5 @@ book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID book getword detCleanupProcessedImfile $pageName camera -var CAMERA - book getword detCleanupProcessedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedImfile $pageName data_state -var CLEANUP_MODE book getword detCleanupProcessedImfile $pageName dbname -var DBNAME @@ -244,5 +244,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -352,5 +352,5 @@ book getword detCleanupProcessedExp $pageName exp_id -var EXP_ID book getword detCleanupProcessedExp $pageName camera -var CAMERA - book getword detCleanupProcessedExp $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedExp $pageName data_state -var CLEANUP_MODE book getword detCleanupProcessedExp $pageName dbname -var DBNAME @@ -362,5 +362,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --exp_id $EXP_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -470,5 +470,5 @@ book getword detCleanupStackedImfile $pageName class_id -var CLASS_ID book getword detCleanupStackedImfile $pageName camera -var CAMERA - book getword detCleanupStackedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupStackedImfile $pageName data_state -var CLEANUP_MODE book getword detCleanupStackedImfile $pageName dbname -var DBNAME @@ -480,5 +480,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -588,5 +588,5 @@ book getword detCleanupNormStatImfile $pageName iteration -var ITERATION book getword detCleanupNormStatImfile $pageName camera -var CAMERA - book getword detCleanupNormStatImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormStatImfile $pageName data_state -var CLEANUP_MODE book getword detCleanupNormStatImfile $pageName dbname -var DBNAME @@ -598,5 +598,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -704,5 +704,5 @@ book getword detCleanupNormImfile $pageName class_id -var CLASS_ID book getword detCleanupNormImfile $pageName camera -var CAMERA - book getword detCleanupNormImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormImfile $pageName data_state -var CLEANUP_MODE book getword detCleanupNormImfile $pageName dbname -var DBNAME @@ -714,5 +714,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -819,5 +819,5 @@ book getword detCleanupNormExp $pageName iteration -var ITERATION book getword detCleanupNormExp $pageName camera -var CAMERA - book getword detCleanupNormExp $pageName state -var CLEANUP_MODE + book getword detCleanupNormExp $pageName data_state -var CLEANUP_MODE book getword detCleanupNormExp $pageName dbname -var DBNAME @@ -829,5 +829,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -937,5 +937,5 @@ book getword detCleanupResidImfile $pageName iteration -var ITERATION book getword detCleanupResidImfile $pageName camera -var CAMERA - book getword detCleanupResidImfile $pageName state -var CLEANUP_MODE + book getword detCleanupResidImfile $pageName data_state -var CLEANUP_MODE book getword detCleanupResidImfile $pageName dbname -var DBNAME @@ -947,5 +947,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -1056,5 +1056,5 @@ book getword detCleanupResidExp $pageName iteration -var ITERATION book getword detCleanupResidExp $pageName camera -var CAMERA - book getword detCleanupResidExp $pageName state -var CLEANUP_MODE + book getword detCleanupResidExp $pageName data_state -var CLEANUP_MODE book getword detCleanupResidExp $pageName dbname -var DBNAME @@ -1066,5 +1066,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 25323) +++ trunk/ippTools/share/Makefile.am (revision 25324) @@ -47,4 +47,5 @@ dettool_normalizedstat.sql \ dettool_pending.sql \ + dettool_pendingcleanup_detrunsummary.sql \ dettool_pendingcleanup_normalizedexp.sql \ dettool_pendingcleanup_normalizedimfile.sql \ Index: trunk/ippTools/share/camtool_pendingcleanupexp.sql =================================================================== --- trunk/ippTools/share/camtool_pendingcleanupexp.sql (revision 25323) +++ trunk/ippTools/share/camtool_pendingcleanupexp.sql (revision 25324) @@ -13,4 +13,4 @@ USING(cam_id) WHERE - (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged') + (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged' OR camRun.state = 'goto_scrubbed') Index: trunk/ippTools/share/camtool_pendingcleanuprun.sql =================================================================== --- trunk/ippTools/share/camtool_pendingcleanuprun.sql (revision 25323) +++ trunk/ippTools/share/camtool_pendingcleanuprun.sql (revision 25324) @@ -9,3 +9,3 @@ USING (exp_id) WHERE - (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged') + (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged' OR camRun.state = 'goto_scrubbed') Index: trunk/ippTools/share/dettool_donecleanup_normalizedexp.sql =================================================================== --- trunk/ippTools/share/dettool_donecleanup_normalizedexp.sql (revision 25324) +++ trunk/ippTools/share/dettool_donecleanup_normalizedexp.sql (revision 25324) @@ -0,0 +1,11 @@ +SELECT + detNormalizedExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedExp + USING(det_id,iteration) +WHERE + (detNormalizedExp.data_state = 'cleaned' + OR detNormalizedExp.data_state = 'scrubbed' + OR detNormalizedExp.data_state = 'purged') + Index: trunk/ippTools/share/dettool_donecleanup_normalizedimfile.sql =================================================================== --- trunk/ippTools/share/dettool_donecleanup_normalizedimfile.sql (revision 25324) +++ trunk/ippTools/share/dettool_donecleanup_normalizedimfile.sql (revision 25324) @@ -0,0 +1,11 @@ +SELECT + detNormalizedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedImfile + USING(det_id,iteration) +WHERE + (detNormalizedImfile.data_state = 'cleaned' + OR detNormalizedImfile.data_state = 'scrubbed' + OR detNormalizedImfile.data_state = 'purged') + Index: trunk/ippTools/share/dettool_donecleanup_normalizedstat.sql =================================================================== --- trunk/ippTools/share/dettool_donecleanup_normalizedstat.sql (revision 25324) +++ trunk/ippTools/share/dettool_donecleanup_normalizedstat.sql (revision 25324) @@ -0,0 +1,11 @@ +SELECT + detNormalizedStatImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedStatImfile + USING(det_id,iteration) +WHERE + (detNormalizedStatImfile.data_state = 'cleaned' + OR detNormalizedStatImfile.data_state = 'scrubbed' + OR detNormalizedStatImfile.data_state = 'purged') + Index: trunk/ippTools/share/dettool_donecleanup_processedexp.sql =================================================================== --- trunk/ippTools/share/dettool_donecleanup_processedexp.sql (revision 25324) +++ trunk/ippTools/share/dettool_donecleanup_processedexp.sql (revision 25324) @@ -0,0 +1,11 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT + detProcessedExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detProcessedExp + USING(det_id) +WHERE + (detProcessedExp.data_state = 'cleaned' + OR detProcessedExp.data_state = 'scrubbed' + OR detProcessedExp.data_state = 'purged') Index: trunk/ippTools/share/dettool_donecleanup_processedimfile.sql =================================================================== --- trunk/ippTools/share/dettool_donecleanup_processedimfile.sql (revision 25324) +++ trunk/ippTools/share/dettool_donecleanup_processedimfile.sql (revision 25324) @@ -0,0 +1,11 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT + detProcessedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detProcessedImfile + USING(det_id) +WHERE + (detProcessedImfile.data_state = 'cleaned' + OR detProcessedImfile.data_state = 'scrubbed' + OR detProcessedImfile.data_state = 'purged') Index: trunk/ippTools/share/dettool_donecleanup_residexp.sql =================================================================== --- trunk/ippTools/share/dettool_donecleanup_residexp.sql (revision 25324) +++ trunk/ippTools/share/dettool_donecleanup_residexp.sql (revision 25324) @@ -0,0 +1,10 @@ +SELECT + detResidExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detResidExp + USING(det_id,iteration) +WHERE + (detResidExp.data_state = 'cleaned' + OR detResidExp.data_state = 'scrubbed' + OR detResidExp.data_state = 'purged') Index: trunk/ippTools/share/dettool_donecleanup_residimfile.sql =================================================================== --- trunk/ippTools/share/dettool_donecleanup_residimfile.sql (revision 25324) +++ trunk/ippTools/share/dettool_donecleanup_residimfile.sql (revision 25324) @@ -0,0 +1,12 @@ +SELECT + detResidImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detResidImfile + USING(det_id,iteration) +WHERE + (detResidImfile.data_state = 'cleaned' + OR detResidImfile.data_state = 'scrubbed' + OR detResidImfile.data_state = 'purged') + + Index: trunk/ippTools/share/dettool_donecleanup_stacked.sql =================================================================== --- trunk/ippTools/share/dettool_donecleanup_stacked.sql (revision 25324) +++ trunk/ippTools/share/dettool_donecleanup_stacked.sql (revision 25324) @@ -0,0 +1,10 @@ +SELECT + detStackedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detStackedImfile + USING(det_id,iteration) +WHERE + (detStackedImfile.data_state = 'cleaned' + OR detStackedImfile.data_state = 'scrubbed' + OR detStackedImfile.data_state = 'purged') Index: trunk/ippTools/share/dettool_pendingcleanup_detrunsummary.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_detrunsummary.sql (revision 25324) +++ trunk/ippTools/share/dettool_pendingcleanup_detrunsummary.sql (revision 25324) @@ -0,0 +1,42 @@ +SELECT DISTINCT detRunSummary.* +FROM detRunSummary + JOIN detProcessedImfile USING (det_id) + JOIN detProcessedExp USING (det_id) + JOIN detStackedImfile USING (det_id,iteration) + JOIN detResidExp USING (det_id,iteration) + JOIN detResidImfile USING (det_id,iteration) + JOIN detNormalizedStatImfile USING (det_id,iteration) + JOIN detNormalizedImfile USING (det_id,iteration) + JOIN detNormalizedExp USING (det_id,iteration) +WHERE (!( + ( + detRunSummary.data_state = 'full' OR + detRunSummary.data_state = 'cleaned' + ) OR + ( + !((detProcessedImfile.data_state = 'cleaned' OR + detProcessedImfile.data_state = 'scrubbed' OR + detProcessedImfile.data_state = 'purged') AND + (detProcessedExp.data_state = 'cleaned' OR + detProcessedExp.data_state = 'scrubbed' OR + detProcessedExp.data_state = 'purged') AND + (detStackedImfile.data_state = 'cleaned' OR + detStackedImfile.data_state = 'scrubbed' OR + detStackedImfile.data_state = 'purged') AND + (detResidExp.data_state = 'cleaned' OR + detResidExp.data_state = 'scrubbed' OR + detResidExp.data_state = 'purged') AND + (detResidImfile.data_state = 'cleaned' OR + detResidImfile.data_state = 'scrubbed' OR + detResidImfile.data_state = 'purged') AND + (detNormalizedStatImfile.data_state = 'cleaned' OR + detNormalizedStatImfile.data_state = 'scrubbed' OR + detNormalizedStatImfile.data_state = 'purged') AND + (detNormalizedImfile.data_state = 'cleaned' OR + detNormalizedImfile.data_state = 'scrubbed' OR + detNormalizedImfile.data_state = 'purged') AND + (detNormalizedExp.data_state = 'cleaned' OR + detNormalizedExp.data_state = 'scrubbed' OR + detNormalizedExp.data_state = 'purged') + ) + ))) Index: trunk/ippTools/share/dettool_pendingcleanup_normalizedexp.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_normalizedexp.sql (revision 25323) +++ trunk/ippTools/share/dettool_pendingcleanup_normalizedexp.sql (revision 25324) @@ -1,9 +1,21 @@ SELECT detNormalizedExp.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detNormalizedExp USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detNormalizedExp.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detNormalizedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detNormalizedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detNormalizedExp.data_state = 'full') + OR detNormalizedExp.data_state = 'goto_cleaned' + OR detNormalizedExp.data_state = 'goto_scrubbed' + OR detNormalizedExp.data_state = 'goto_purged') + Index: trunk/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (revision 25323) +++ trunk/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (revision 25324) @@ -1,9 +1,21 @@ SELECT DISTINCT detNormalizedImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detNormalizedImfile USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detNormalizedImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detNormalizedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detNormalizedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detNormalizedImfile.data_state = 'full') + OR detNormalizedImfile.data_state = 'goto_cleaned' + OR detNormalizedImfile.data_state = 'goto_scrubbed' + OR detNormalizedImfile.data_state = 'goto_purged') + Index: trunk/ippTools/share/dettool_pendingcleanup_normalizedstat.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_normalizedstat.sql (revision 25323) +++ trunk/ippTools/share/dettool_pendingcleanup_normalizedstat.sql (revision 25324) @@ -1,9 +1,21 @@ -SELECT +SELECT DISTINCT detNormalizedStatImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detNormalizedStatImfile USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detNormalizedStatImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detNormalizedStatImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detNormalizedStatImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detNormalizedStatImfile.data_state = 'full') + OR detNormalizedStatImfile.data_state = 'goto_cleaned' + OR detNormalizedStatImfile.data_state = 'goto_scrubbed' + OR detNormalizedStatImfile.data_state = 'goto_purged') + Index: trunk/ippTools/share/dettool_pendingcleanup_processedexp.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_processedexp.sql (revision 25323) +++ trunk/ippTools/share/dettool_pendingcleanup_processedexp.sql (revision 25324) @@ -2,9 +2,18 @@ SELECT detProcessedExp.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detProcessedExp USING(det_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detProcessedExp.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detProcessedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detProcessedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detProcessedExp.data_state = 'full') + OR detProcessedExp.data_state = 'goto_cleaned' + OR detProcessedExp.data_state = 'goto_scrubbed' + OR detProcessedExp.data_state = 'goto_purged') Index: trunk/ippTools/share/dettool_pendingcleanup_processedimfile.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_processedimfile.sql (revision 25323) +++ trunk/ippTools/share/dettool_pendingcleanup_processedimfile.sql (revision 25324) @@ -1,10 +1,15 @@ --- need to restrict to a single detRunSummary (require all to say 'cleaned'?) SELECT detProcessedImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detProcessedImfile USING(det_id) +JOIN detInputExp + USING(det_id,iteration,exp_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detProcessedImfile.data_state = 'full' + (detProcessedImfile.data_state = 'goto_cleaned' + OR detProcessedImfile.data_state = 'goto_scrubbed' + OR detProcessedImfile.data_state = 'goto_purged') + Index: trunk/ippTools/share/dettool_pendingcleanup_residexp.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_residexp.sql (revision 25323) +++ trunk/ippTools/share/dettool_pendingcleanup_residexp.sql (revision 25324) @@ -1,9 +1,20 @@ SELECT detResidExp.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detResidExp USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration,exp_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detResidExp.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detResidExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detResidExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detResidExp.data_state = 'full') + OR detResidExp.data_state = 'goto_cleaned' + OR detResidExp.data_state = 'goto_scrubbed' + OR detResidExp.data_state = 'goto_purged') Index: trunk/ippTools/share/dettool_pendingcleanup_residimfile.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_residimfile.sql (revision 25323) +++ trunk/ippTools/share/dettool_pendingcleanup_residimfile.sql (revision 25324) @@ -1,9 +1,22 @@ SELECT detResidImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detResidImfile USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration,exp_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detResidImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detResidImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detResidImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detResidImfile.data_state = 'full') + OR detResidImfile.data_state = 'goto_cleaned' + OR detResidImfile.data_state = 'goto_scrubbed' + OR detResidImfile.data_state = 'goto_purged') + + Index: trunk/ippTools/share/dettool_pendingcleanup_stacked.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_stacked.sql (revision 25323) +++ trunk/ippTools/share/dettool_pendingcleanup_stacked.sql (revision 25324) @@ -1,9 +1,20 @@ SELECT DISTINCT detStackedImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detStackedImfile - USING(det_id,iteration) + USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detStackedImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detStackedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detStackedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detStackedImfile.data_state = 'full') + OR detStackedImfile.data_state = 'goto_cleaned' + OR detStackedImfile.data_state = 'goto_scrubbed' + OR detStackedImfile.data_state = 'goto_purged') Index: trunk/ippTools/share/faketool_pendingcleanupimfile.sql =================================================================== --- trunk/ippTools/share/faketool_pendingcleanupimfile.sql (revision 25323) +++ trunk/ippTools/share/faketool_pendingcleanupimfile.sql (revision 25324) @@ -13,3 +13,7 @@ USING(fake_id) WHERE - fakeRun.state = 'goto_cleaned' + ((fakeRun.state = 'goto_cleaned' AND fakeProcessedImfile.data_state = 'full') +OR + (fakeRun.state = 'goto_scrubbed' AND fakeProcessedImfile.data_state = 'full') +OR + (fakeRun.state = 'goto_purged' AND fakeProcessedImfile.data_state != 'purged')) Index: trunk/ippTools/share/faketool_pendingcleanuprun.sql =================================================================== --- trunk/ippTools/share/faketool_pendingcleanuprun.sql (revision 25323) +++ trunk/ippTools/share/faketool_pendingcleanuprun.sql (revision 25324) @@ -11,3 +11,4 @@ USING (exp_id) WHERE - fakeRun.state = 'goto_cleaned' + (fakeRun.state = 'goto_cleaned' OR fakeRun.state = 'goto_scrubbed' OR fakeRun.state = 'goto_purged') + Index: trunk/ippTools/src/chiptool.c =================================================================== --- trunk/ippTools/src/chiptool.c (revision 25323) +++ trunk/ippTools/src/chiptool.c (revision 25324) @@ -54,4 +54,5 @@ static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); +static bool toscrubbedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -93,4 +94,5 @@ MODECASE(CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + MODECASE(CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(CHIPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(CHIPTOOL_MODE_IMPORTRUN, importrunMode); @@ -1333,5 +1335,8 @@ return change_imfile_data_state(config, "purged", "goto_purged"); } - +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} bool exportrunMode(pxConfig *config) { Index: trunk/ippTools/src/chiptool.h =================================================================== --- trunk/ippTools/src/chiptool.h (revision 25323) +++ trunk/ippTools/src/chiptool.h (revision 25324) @@ -45,4 +45,5 @@ CHIPTOOL_MODE_TOFULLIMFILE, CHIPTOOL_MODE_TOPURGEDIMFILE, + CHIPTOOL_MODE_TOSCRUBBEDIMFILE, CHIPTOOL_MODE_EXPORTRUN, CHIPTOOL_MODE_IMPORTRUN Index: trunk/ippTools/src/chiptoolConfig.c =================================================================== --- trunk/ippTools/src/chiptoolConfig.c (revision 25323) +++ trunk/ippTools/src/chiptoolConfig.c (revision 25324) @@ -248,4 +248,9 @@ psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -282,4 +287,5 @@ PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + PXOPT_ADD_MODE("-toscrubbedimfile", "set imfile state to scrubbed", CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", CHIPTOOL_MODE_EXPORTRUN, exportrunArgs); Index: trunk/ippTools/src/dettool.c =================================================================== --- trunk/ippTools/src/dettool.c (revision 25323) +++ trunk/ippTools/src/dettool.c (revision 25324) @@ -150,4 +150,5 @@ MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); MODECASE(DETTOOL_MODE_RERUN, rerunMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryMode); // register MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); @@ -1827,4 +1828,15 @@ if (!strcmp(data_state, "drop")) return true; if (!strcmp(data_state, "register")) return true; + // These are valid data states, and are necessary for the cleanup to work correctly. + if (!strcmp(data_state, "full")) return true; + if (!strcmp(data_state, "goto_cleaned")) return true; + if (!strcmp(data_state, "goto_scrubbed")) return true; + if (!strcmp(data_state, "goto_purged")) return true; + if (!strcmp(data_state, "cleaned")) return true; + if (!strcmp(data_state, "scrubbed")) return true; + if (!strcmp(data_state, "purged")) return true; + if (!strcmp(data_state, "error_cleaned")) return true; + if (!strcmp(data_state, "error_scrubbed")) return true; + if (!strcmp(data_state, "error_purged")) return true; psError(PS_ERR_UNKNOWN, true, "invalid data state: %s", data_state); @@ -1853,5 +1865,5 @@ if (!isValidDataState (data_state)) return false; - + char *query = "UPDATE detProcessedImfile SET data_state = '%s'" " WHERE det_id = %" PRId64 @@ -1900,5 +1912,5 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, @@ -1922,6 +1934,7 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration)) { + " AND class_id = '%s'"; +/* fprintf(stderr,"DETTOOL SAYS: %s\n",query); */ + if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration,class_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for det_id %" PRId64 ", iteration %" PRId32, @@ -1944,5 +1957,5 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, Index: trunk/ippTools/src/dettool.h =================================================================== --- trunk/ippTools/src/dettool.h (revision 25323) +++ trunk/ippTools/src/dettool.h (revision 25324) @@ -111,4 +111,5 @@ DETTOOL_MODE_REVERTDETRUNSUMMARY, DETTOOL_MODE_UPDATEDETRUNSUMMARY, + DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, DETTOOL_MODE_UPDATEDETRUN, DETTOOL_MODE_RERUN, @@ -209,4 +210,5 @@ bool revertdetrunsummaryMode(pxConfig *config); bool updatedetrunsummaryMode(pxConfig *config); +bool pendingcleanup_detrunsummaryMode(pxConfig *config); // other utilities Index: trunk/ippTools/src/dettoolConfig.c =================================================================== --- trunk/ippTools/src/dettoolConfig.c (revision 25323) +++ trunk/ippTools/src/dettoolConfig.c (revision 25324) @@ -41,11 +41,11 @@ // XXX EAM : is this used? does it make sense? psMetadata *pendingArgs = psMetadataAlloc(); - psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri", 0, "search by URL", NULL); - psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -definebytag @@ -70,5 +70,4 @@ psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); - psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); @@ -126,4 +125,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_max", 0, "define max solar angle", NAN); @@ -139,12 +139,4 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); - - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_angle_min", 0, "define min moon angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_angle_max", 0, "define max moon angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_alt_min", 0, "define min moon alt", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_alt_max", 0, "define max moon alt", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_min", 0, "define min moon phase", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max", 0, "define max moon phase", NAN); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); @@ -217,7 +209,7 @@ // -addcorrectimfile psMetadata *addcorrectimfileArgs = psMetadataAlloc(); - psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); - psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); - psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); + psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); @@ -316,5 +308,5 @@ psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_processedimfile @@ -376,5 +368,5 @@ psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); - psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_processedexp @@ -436,5 +428,5 @@ psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); - psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_stacked @@ -488,5 +480,5 @@ psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedstat @@ -550,5 +542,5 @@ psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedimfile @@ -608,5 +600,5 @@ psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); - psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedexp @@ -688,5 +680,5 @@ psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_residimfile @@ -769,5 +761,5 @@ psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject", 0, "exposure is not to be stacked in the next iteration", false); - psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_residexp @@ -823,4 +815,13 @@ psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, "declare that this detrun iteration is accepted as a master", false); psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, "reject this detrun iteration as a master", false); + psMetadataAddStr(updatedetrunsummaryArgs, PS_LIST_TAIL, "-data_state", 0, "set the data state", NULL); + psMetadataAddS32(updatedetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + + // -pendingcleanup_detrunsummary + psMetadata *pendingcleanup_detrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddU64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updatedetrun @@ -978,4 +979,6 @@ PXOPT_ADD_MODE("-updatedetrunsummary", "", DETTOOL_MODE_UPDATEDETRUNSUMMARY, updatedetrunsummaryArgs); + PXOPT_ADD_MODE("-pendingcleanup_detrunsummary", "", DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryArgs); + PXOPT_ADD_MODE("-updatedetrun", "", DETTOOL_MODE_UPDATEDETRUN, updatedetrunArgs); PXOPT_ADD_MODE("-rerun", "", DETTOOL_MODE_RERUN, rerunArgs); Index: trunk/ippTools/src/dettool_detrunsummary.c =================================================================== --- trunk/ippTools/src/dettool_detrunsummary.c (revision 25323) +++ trunk/ippTools/src/dettool_detrunsummary.c (revision 25324) @@ -336,5 +336,65 @@ } -// XXX need to add -data_state here +bool pendingcleanup_detrunsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_detrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return(false); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRunSummary"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + // fprintf(stderr,">>>%s<<<\n",query); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pendingCleanupDetRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return(false); + } + psFree(output); + + return(true); +} + + +// preliminary code now. bool updatedetrunsummaryMode(pxConfig *config) { @@ -344,4 +404,7 @@ PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", ((accept == 0)&&(reject == 0)), false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", ((accept == 0)&&(reject == 0)), false); + if (accept && reject) { @@ -350,13 +413,109 @@ } - if (!(accept || reject)) { - psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required"); - return false; - } - - char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; - if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; + if (!(accept || reject || (data_state != NULL))) { + psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required if -data_state is not supplied"); + return false; + } + + if (accept || reject) { + char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + else { + PS_ASSERT_PTR_NON_NULL(data_state,false); + +/* if (!isValidDataState(data_state)) return false; */ + + + + char *query_detRunSummary = "UPDATE detRunSummary SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detRunSummary, data_state, det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return false; + } + + /* This check allows the one update to flag everything for cleanup. The check for full is only temporary while I test for bugs. */ + if ((!strncmp(data_state,"goto_",5) + // || (!strcmp(data_state,"full")) + // || (!strcmp(data_state,"cleaned")) + )) { + char *query_detProcessedImfile = "UPDATE detProcessedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedImfile,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detProcessedExp = "UPDATE detProcessedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedExp,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedImfile = "UPDATE detNormalizedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedStatImfile = "UPDATE detNormalizedStatImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedStatImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedStatImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedExp = "UPDATE detNormalizedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidImfile = "UPDATE detResidImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidExp = "UPDATE detResidExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detStackedImfile = "UPDATE detStackedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detStackedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detStackedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + } + /* End if */ + } Index: trunk/ippTools/src/dettool_processedimfile.c =================================================================== --- trunk/ippTools/src/dettool_processedimfile.c (revision 25323) +++ trunk/ippTools/src/dettool_processedimfile.c (revision 25324) @@ -314,5 +314,5 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - +/* int i; */ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); @@ -345,4 +345,5 @@ return false; } +/* fprintf(stderr,"DETTOOL:procimfile: %s\n",query); */ psFree(query); @@ -350,8 +351,10 @@ if (!output) { psError(PS_ERR_UNKNOWN, false, "database error"); +/* fprintf(stderr,"WTF !output?\n"); */ return false; } if (!psArrayLength(output)) { psTrace("dettool", PS_LOG_INFO, "no rows found"); +/* fprintf(stderr,"WTF no rows??\n"); */ psFree(output); return true; @@ -359,4 +362,6 @@ // negative simple so the default is true +/* i = (int) ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple); */ +/* fprintf(stderr,">>%d<<\n",i); */ if (!ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); @@ -364,4 +369,6 @@ return false; } +/* fprintf(stderr,"DETTOOL:procimfile: %s\n",output); */ +/* psFree(output); */ psFree(output); Index: trunk/ippTools/src/faketool.c =================================================================== --- trunk/ippTools/src/faketool.c (revision 25323) +++ trunk/ippTools/src/faketool.c (revision 25324) @@ -52,4 +52,5 @@ static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); +static bool toscrubbedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -91,4 +92,5 @@ MODECASE(FAKETOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + MODECASE(FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(FAKETOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(FAKETOOL_MODE_IMPORTRUN, importrunMode); @@ -1251,4 +1253,8 @@ return change_imfile_data_state(config, "purged", "goto_purged"); } +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} bool exportrunMode(pxConfig *config) Index: trunk/ippTools/src/faketool.h =================================================================== --- trunk/ippTools/src/faketool.h (revision 25323) +++ trunk/ippTools/src/faketool.h (revision 25324) @@ -45,4 +45,5 @@ FAKETOOL_MODE_TOFULLIMFILE, FAKETOOL_MODE_TOPURGEDIMFILE, + FAKETOOL_MODE_TOSCRUBBEDIMFILE, FAKETOOL_MODE_EXPORTRUN, FAKETOOL_MODE_IMPORTRUN Index: trunk/ippTools/src/faketoolConfig.c =================================================================== --- trunk/ippTools/src/faketoolConfig.c (revision 25323) +++ trunk/ippTools/src/faketoolConfig.c (revision 25324) @@ -301,4 +301,9 @@ psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "fake ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -337,4 +342,5 @@ PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", FAKETOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + PXOPT_ADD_MODE("-toscrubbedimfile", "set imfile state to scrubbed", FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", FAKETOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", FAKETOOL_MODE_IMPORTRUN, importrunArgs); Index: trunk/ippTools/src/pxtools.c =================================================================== --- trunk/ippTools/src/pxtools.c (revision 25323) +++ trunk/ippTools/src/pxtools.c (revision 25324) @@ -37,4 +37,6 @@ if (!strcmp(state, "goto_cleaned")) return true; if (!strcmp(state, "error_cleaned")) return true; + if (!strcmp(state, "goto_purged")) return true; + if (!strcmp(state, "error_purged")) return true; if (!strcmp(state, "goto_scrubbed")) return true; if (!strcmp(state, "error_scrubbed")) return true; @@ -42,12 +44,11 @@ if (!strcmp(state, "update")) return true; if (!strcmp(state, "purged")) return true; - if (!strcmp(state, "goto_purged")) return true; - if (!strcmp(state, "error_purged")) return true; - + if (!strcmp(state, "scrubbed")) return true; return false; } -// 'scrubbed' is a virtual state equivalent to cleaned, but allows files to be removed -// even if the config files is missing +// 'scrubbed' is no longer a virtual state equivalent to cleaned, but allows files to be removed +// even if the config files is missing. This change was prompted as files that are cleaned can +// be regenerated, but that is not certain after being scrubbed. Index: trunk/ippTools/src/regtool.c =================================================================== --- trunk/ippTools/src/regtool.c (revision 25323) +++ trunk/ippTools/src/regtool.c (revision 25324) @@ -929,6 +929,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); Index: trunk/ippTools/src/warptool.c =================================================================== --- trunk/ippTools/src/warptool.c (revision 25323) +++ trunk/ippTools/src/warptool.c (revision 25324) @@ -53,4 +53,5 @@ static bool tocleanedskyfileMode(pxConfig *config); static bool topurgedskyfileMode(pxConfig *config); +static bool toscrubbedskyfileMode(pxConfig *config); static bool tofullskyfileMode(pxConfig *config); static bool updateskyfileMode(pxConfig *config); @@ -102,4 +103,5 @@ MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); MODECASE(WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); MODECASE(WARPTOOL_MODE_UPDATESKYFILE, updateskyfileMode); @@ -234,6 +236,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-reduction", "fakeRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); @@ -1668,4 +1670,8 @@ return change_skyfile_data_state(config, "purged", "goto_purged"); } +static bool toscrubbedskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); +} static bool updateskyfileMode(pxConfig *config) Index: trunk/ippTools/src/warptool.h =================================================================== --- trunk/ippTools/src/warptool.h (revision 25323) +++ trunk/ippTools/src/warptool.h (revision 25324) @@ -49,4 +49,5 @@ WARPTOOL_MODE_TOCLEANEDSKYFILE, WARPTOOL_MODE_TOPURGEDSKYFILE, + WARPTOOL_MODE_TOSCRUBBEDSKYFILE, WARPTOOL_MODE_TOFULLSKYFILE, WARPTOOL_MODE_UPDATESKYFILE, Index: trunk/ippTools/src/warptoolConfig.c =================================================================== --- trunk/ippTools/src/warptoolConfig.c (revision 25323) +++ trunk/ippTools/src/warptoolConfig.c (revision 25324) @@ -279,4 +279,9 @@ psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + // -toscrubbedskyfile + psMetadata *toscrubbedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + // -tofullskyfile psMetadata *tofullskyfileArgs = psMetadataAlloc(); @@ -328,4 +333,5 @@ PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs); PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); + PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs); Index: trunk/ppImage/src/ppImageOptions.c =================================================================== --- trunk/ppImage/src/ppImageOptions.c (revision 25323) +++ trunk/ppImage/src/ppImageOptions.c (revision 25324) @@ -66,5 +66,5 @@ options->markValue = 0x00; // A safe bit for internal marking options->burntoolMask = 0x00; // Suspect pixels that fall where a burntool trail is expected. - options->burntoolTrails = 0x00; // Which types of burntool areas to mask. + options->burntoolTrails = 0x07; // Which types of burntool areas to mask. // crosstalk options options->doCrosstalkMeasure = false; // measure crosstalk @@ -248,5 +248,6 @@ options->applyParity = psMetadataLookupBool(NULL, recipe, "APPLY.CELL.PARITY"); - options->burntoolTrails = psMetadataLookupU16(&status, recipe, "BURNTOOL.TRAILS"); + options->burntoolTrails = psMetadataLookupS32(&status, recipe, "BURNTOOL.TRAILS"); + fprintf(stderr,"TRAILS: %d %d %d\n",options->burntoolTrails,psMetadataLookupS32(&status,recipe,"BURNTOOL.TRAILS"),status); if (!status) { psWarning("BURNTOOL.TRAILS not found in recipe: setting to default value.\n"); Index: trunk/psModules/src/astrom/pmAstrometryWCS.c =================================================================== --- trunk/psModules/src/astrom/pmAstrometryWCS.c (revision 25323) +++ trunk/psModules/src/astrom/pmAstrometryWCS.c (revision 25324) @@ -290,5 +290,4 @@ if (pcKeys) { wcs->wcsCDkeys = 0; - wcs->cdelt1 = psMetadataLookupF64 (&status, header, "CDELT1"); wcs->cdelt2 = psMetadataLookupF64 (&status, header, "CDELT2"); @@ -379,5 +378,4 @@ // XXX make it optional to write out CDi_j terms, or other versions // apply CDELT1,2 (degrees / pixel) to yield PCi,j terms of order unity - if (!(wcs->wcsCDkeys)) { @@ -423,4 +421,5 @@ } if (wcs->wcsCDkeys) { + psMetadataAddF64 (header, PS_LIST_TAIL, "CD1_1", PS_META_REPLACE, "", wcs->trans->x->coeff[1][0]); psMetadataAddF64 (header, PS_LIST_TAIL, "CD1_2", PS_META_REPLACE, "", wcs->trans->x->coeff[0][1]); Index: trunk/tools/ipp_apply_burntool.pl =================================================================== --- trunk/tools/ipp_apply_burntool.pl (revision 25323) +++ trunk/tools/ipp_apply_burntool.pl (revision 25324) @@ -140,5 +140,6 @@ my $tempfile = new File::Temp ( TEMPLATE => "$basename.XXXX", DIR => '/tmp', - UNLINK => !$save_temps); + UNLINK => !$save_temps, + SUFFIX => '.fits'); my $tmpImfileReal = $tempfile->filename; # print "tmpImfile: $tmpImfile -> $tmpImfileReal\n";