Index: /branches/eam_branches/ipp-20100621/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/ipp-20100621/dbconfig/changes.txt (revision 28438) +++ /branches/eam_branches/ipp-20100621/dbconfig/changes.txt (revision 28439) @@ -1784,4 +1784,5 @@ quality SMALLINT NOT NULL, -- bad quality flag software_ver VARCHAR(16), -- software version for run + magicked BIGINT NOT NULL DEFAULT 0, -- magic mask applied PRIMARY KEY(diff_phot_id, skycell_id), KEY(fault), Index: /branches/eam_branches/ipp-20100621/dbconfig/diffphot.md =================================================================== --- /branches/eam_branches/ipp-20100621/dbconfig/diffphot.md (revision 28438) +++ /branches/eam_branches/ipp-20100621/dbconfig/diffphot.md (revision 28439) @@ -21,3 +21,4 @@ quality S16 0 # Key software_ver STR 16 + magicked S64 0 END Index: /branches/eam_branches/ipp-20100621/ippMonitor/Makefile.in =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/Makefile.in (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippMonitor/Makefile.in (revision 28439) @@ -16,5 +16,7 @@ $(DESTBIN)/czartool_getLabels.pl \ $(DESTBIN)/czartool_checkServer.pl \ -$(DESTBIN)/czartool_getServerStatus.pl +$(DESTBIN)/czartool_getServerStatus.pl \ +$(DESTBIN)/build_histogram.dvo \ + RAWSRC = \ @@ -50,4 +52,6 @@ $(DESTWWW)/czartool_labels.php \ $(DESTWWW)/czartool_servers.php \ +$(DESTWWW)/histogram.php \ +$(DESTWWW)/show_and_delete_image.php \ Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/histogram.php =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/raw/histogram.php (revision 28439) +++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/histogram.php (revision 28439) @@ -0,0 +1,472 @@ + Histogram

"; + +// set up the form +echo "
\n"; + +$restricted = 0; + +// define restrictiosn to the queries +// ** TABLE RESTRICTIONS ** +$WHERE = "WHERE camRun.state = 'full' AND camRun.chip_id = chipRun.chip_id AND chipRun.exp_id = rawExp.exp_id AND camProcessedExp.cam_id = camRun.cam_id AND camProcessedExp.fault = 0"; +$WHERE = check_restrict ('rawExp.exp_name', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('chipRun.label', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('chipRun.data_group', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('chipRun.dist_group', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('rawExp.filter', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_uq', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_uq', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_uq', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_lq', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_lq', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_lq', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_uq', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_uq', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_uq', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_lq', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_lq', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_lq', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_major', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_major', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_major', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_minor', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_minor', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_minor', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m2', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m2', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m2', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m3', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m3', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m3', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m4', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m4', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m4', $WHERE, 'max', 1.0); +$WHERE = check_ordering ('', $WHERE); + +if ($restricted == 0) { + if ("none" != "none") { + $WHERE = "$WHERE none"; + } +} + +// get the result table count +if (basic == "basic") { + $sql = "SELECT count(*) FROM camRun, chipRun, rawExp, camProcessedExp $WHERE"; +} +if (basic == "summary") { + $sql = "SELECT count(*) FROM (SELECT rawExp.exp_name,chipRun.label,chipRun.data_group,chipRun.dist_group,rawExp.dateobs,rawExp.filter,rawExp.exp_time,rawExp.airmass,camProcessedExp.fwhm_major,camProcessedExp.fwhm_minor,camProcessedExp.fwhm_major_uq,camProcessedExp.fwhm_major_lq,camProcessedExp.fwhm_minor_uq,camProcessedExp.fwhm_minor_lq,camProcessedExp.iq_fwhm_major,camProcessedExp.iq_fwhm_minor,camProcessedExp.iq_m2,camProcessedExp.iq_m3,camProcessedExp.iq_m4 FROM camRun, chipRun, rawExp, camProcessedExp $WHERE) as TEMP"; +} + +$qry = $db->query($sql); +if (dberror($qry)) { + echo "error reading camRun, chipRun, rawExp, camProcessedExp table count
\n"; + echo "
count query : $sql
\n"; + menu_end(); +} +if (!$qry->fetchInto($row)) { + echo "error reading camRun, chipRun, rawExp, camProcessedExp table count
\n"; + echo "
count query : $sql
\n"; + menu_end(); +} +// set up the row counter variables +if ($ID['from']) { + $rowStart = $ID['from']; +} else { + $rowStart = 0; +} +$rowLast = $rowStart + $dTABLE; +$rowTotal = $row[0]; +if ($rowLast > $rowTotal) { $rowLast = $rowTotal; } +echo " $rowStart to $rowLast of $rowTotal items
\n"; + +// query the database +if (basic == "basic") { + $sql = "SELECT rawExp.exp_name,chipRun.label,chipRun.data_group,chipRun.dist_group,rawExp.dateobs,rawExp.filter,rawExp.exp_time,rawExp.airmass,camProcessedExp.fwhm_major,camProcessedExp.fwhm_minor,camProcessedExp.fwhm_major_uq,camProcessedExp.fwhm_major_lq,camProcessedExp.fwhm_minor_uq,camProcessedExp.fwhm_minor_lq,camProcessedExp.iq_fwhm_major,camProcessedExp.iq_fwhm_minor,camProcessedExp.iq_m2,camProcessedExp.iq_m3,camProcessedExp.iq_m4 FROM camRun, chipRun, rawExp, camProcessedExp $WHERE"; +} +if (basic == "summary") { + $sql = "SELECT rawExp.exp_name,chipRun.label,chipRun.data_group,chipRun.dist_group,rawExp.dateobs,rawExp.filter,rawExp.exp_time,rawExp.airmass,camProcessedExp.fwhm_major,camProcessedExp.fwhm_minor,camProcessedExp.fwhm_major_uq,camProcessedExp.fwhm_major_lq,camProcessedExp.fwhm_minor_uq,camProcessedExp.fwhm_minor_lq,camProcessedExp.iq_fwhm_major,camProcessedExp.iq_fwhm_minor,camProcessedExp.iq_m2,camProcessedExp.iq_m3,camProcessedExp.iq_m4 FROM camRun, chipRun, rawExp, camProcessedExp $WHERE"; +} + +$qry = $db->query($sql); +if (dberror($qry)) { + echo "error reading camRun, chipRun, rawExp, camProcessedExp table
\n"; + echo "
table query : $sql
\n"; + menu_end(); +} + +// ** HEAD CODE ** + +// ** BUTTON RESTRICTIONS ** +$buttonLink = button_restrict_string ('rawExp.exp_name', $buttonLink); +$buttonLink = button_restrict_string ('chipRun.label', $buttonLink); +$buttonLink = button_restrict_string ('chipRun.data_group', $buttonLink); +$buttonLink = button_restrict_string ('chipRun.dist_group', $buttonLink); +$buttonLink = button_restrict_string ('rawExp.dateobs', $buttonLink); +$buttonLink = button_restrict_min ('rawExp.dateobs', $buttonLink); +$buttonLink = button_restrict_max ('rawExp.dateobs', $buttonLink); +$buttonLink = button_restrict_string ('rawExp.filter', $buttonLink); +$buttonLink = button_restrict_string ('rawExp.exp_time', $buttonLink); +$buttonLink = button_restrict_min ('rawExp.exp_time', $buttonLink); +$buttonLink = button_restrict_max ('rawExp.exp_time', $buttonLink); +$buttonLink = button_restrict_string ('rawExp.airmass', $buttonLink); +$buttonLink = button_restrict_min ('rawExp.airmass', $buttonLink); +$buttonLink = button_restrict_max ('rawExp.airmass', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_major', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_major', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_major', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_minor', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_minor', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_minor', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_major_uq', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_major_uq', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_major_uq', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_major_lq', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_major_lq', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_major_lq', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_minor_uq', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_minor_uq', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_minor_uq', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_minor_lq', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_minor_lq', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_minor_lq', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.iq_fwhm_major', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.iq_fwhm_major', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.iq_fwhm_major', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.iq_fwhm_minor', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.iq_fwhm_minor', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.iq_fwhm_minor', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.iq_m2', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.iq_m2', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.iq_m2', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.iq_m3', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.iq_m3', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.iq_m3', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.iq_m4', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.iq_m4', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.iq_m4', $buttonLink); + +$buttonLink = button_restrict_checkbox ('hist.exp_time', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.airmass', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_major_psf', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_minor_psf', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_uq', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_lq', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_major_moments', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_minor_moments', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_m2_moment', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_m3_moment', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_m4_moment', $buttonLink); + +$buttonLink = button_restrict_string ('hist_delta.exp_time', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.airmass', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_major_psf', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_minor_psf', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_uq', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_lq', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_major_moments', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_minor_moments', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_m2_moment', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_m3_moment', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_m4_moment', $buttonLink); + +navigate_buttons ($rowStart, $rowLast, $dTABLE, $rowTotal, $buttonLink, $ID, 'histogram.php'); + +echo "  :   or enter start row: \n"; + +// set up the table +echo "\n"; + +// echo "\n"; // first field is a label set below for the query rows only +// ** TABLE HEADER ** +echo "\n"; +write_header_cell ("list", "Exp Name"); +write_header_cell ("list", "Label"); +write_header_cell ("list", "data grp"); +write_header_cell ("list", "dist grp"); +write_header_cell ("list", "Date/Time"); +write_header_cell ("list", "FILTER"); +write_header_cell ("list", "exp_time "); +write_header_cell ("list", "airmass "); +write_header_cell ("list", "fwhm major psf"); +write_header_cell ("list", "fwhm minor psf"); +write_header_cell ("list", "fwhm UQ"); +write_header_cell ("list", "fwhm LQ"); +write_header_cell ("list", "fwhm major moments"); +write_header_cell ("list", "fwhm minor moments"); +write_header_cell ("list", "m2 moment"); +write_header_cell ("list", "m3 moment"); +write_header_cell ("list", "m4 moment"); +echo "\n"; + +// query restriction form +// echo "\n"; +// echo "\n"; +// ** TABLE QUERY ** +echo "\n"; +write_query_row ('rawExp.exp_name', 5, 'string'); +write_query_row ('chipRun.label', 7, 'string'); +write_query_row ('chipRun.data_group', 7, 'string'); +write_query_row ('chipRun.dist_group', 7, 'string'); +write_query_row ('rawExp.dateobs', 19, 'min'); +write_query_row ('rawExp.filter', 10, 'string'); +write_query_row ('rawExp.exp_time', 5, 'min'); +write_query_row ('rawExp.airmass', 5, 'min'); +write_query_row ('camProcessedExp.fwhm_major', 5, 'min'); +write_query_row ('camProcessedExp.fwhm_minor', 5, 'min'); +write_query_row ('camProcessedExp.fwhm_major_uq', 5, 'min'); +write_query_row ('camProcessedExp.fwhm_major_lq', 5, 'min'); +write_query_row ('camProcessedExp.iq_fwhm_major', 5, 'min'); +write_query_row ('camProcessedExp.iq_fwhm_minor', 5, 'min'); +write_query_row ('camProcessedExp.iq_m2', 5, 'min'); +write_query_row ('camProcessedExp.iq_m3', 5, 'min'); +write_query_row ('camProcessedExp.iq_m4', 5, 'min'); +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +write_query_row ('rawExp.dateobs', 19, 'max'); +echo "\n"; +write_query_row ('rawExp.exp_time', 5, 'max'); +write_query_row ('rawExp.airmass', 5, 'max'); +write_query_row ('camProcessedExp.fwhm_major', 5, 'max'); +write_query_row ('camProcessedExp.fwhm_minor', 5, 'max'); +write_query_row ('camProcessedExp.fwhm_major_uq', 5, 'max'); +write_query_row ('camProcessedExp.fwhm_major_lq', 5, 'max'); +write_query_row ('camProcessedExp.iq_fwhm_major', 5, 'max'); +write_query_row ('camProcessedExp.iq_fwhm_minor', 5, 'max'); +write_query_row ('camProcessedExp.iq_m2', 5, 'max'); +write_query_row ('camProcessedExp.iq_m3', 5, 'max'); +write_query_row ('camProcessedExp.iq_m4', 5, 'max'); +echo "\n"; +// echo "\n"; + +//checkboxes +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n";//dateobs +echo "\n"; +write_query_checkbox ('hist.exp_time', 'Histogram?'); +write_query_checkbox ('hist.airmass', 'Histogram?'); +write_query_checkbox ('hist.fwhm_major_psf', 'Histogram?'); +write_query_checkbox ('hist.fwhm_minor_psf', 'Histogram?'); +write_query_checkbox ('hist.fwhm_uq', 'Histogram?'); +write_query_checkbox ('hist.fwhm_lq', 'Histogram?'); +write_query_checkbox ('hist.fwhm_major_moments', 'Histogram?'); +write_query_checkbox ('hist.fwhm_minor_moments', 'Histogram?'); +write_query_checkbox ('hist.fwhm_m2_moment', 'Histogram?'); +write_query_checkbox ('hist.fwhm_m3_moment', 'Histogram?'); +write_query_checkbox ('hist.fwhm_m4_moment', 'Histogram?'); +echo "\n"; + +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +write_query_row ('hist.exp_time_delta', 5, 'string'); +write_query_row ('hist.airmass_delta', 5, 'string'); +write_query_row ('hist.fwhm_major_psf_delta', 5, 'string'); +write_query_row ('hist.fwhm_minor_psf_delta', 5, 'string'); +write_query_row ('hist.fwhm_uq_delta', 5, 'string'); +write_query_row ('hist.fwhm_lq_delta', 5, 'string'); +write_query_row ('hist.fwhm_major_moments_delta', 5, 'string'); +write_query_row ('hist.fwhm_minor_moments_delta', 5, 'string'); +write_query_row ('hist.fwhm_m2_moment_delta', 5, 'string'); +write_query_row ('hist.fwhm_m3_moment_delta', 5, 'string'); +write_query_row ('hist.fwhm_m4_moment_delta', 5, 'string'); +echo "\n"; + +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +write_query_row ('hist.exp_time_count', 5, 'string'); +write_query_row ('hist.airmass_count', 5, 'string'); +write_query_row ('hist.fwhm_major_psf_count', 5, 'string'); +write_query_row ('hist.fwhm_minor_psf_count', 5, 'string'); +write_query_row ('hist.fwhm_uq_count', 5, 'string'); +write_query_row ('hist.fwhm_lq_count', 5, 'string'); +write_query_row ('hist.fwhm_major_moments_count', 5, 'string'); +write_query_row ('hist.fwhm_minor_moments_count', 5, 'string'); +write_query_row ('hist.fwhm_m2_moment_count', 5, 'string'); +write_query_row ('hist.fwhm_m3_moment_count', 5, 'string'); +write_query_row ('hist.fwhm_m4_moment_count', 5, 'string'); +echo "\n"; + +// close the table and form +echo "
≥
≤          
           
          Bin size (delta)
          Bins number (default: 100)
\n"; +echo "About the \"good\" number of bins

\n"; +echo "\n"; +$pass = $ID['pass']; +$proj = $ID['proj']; +$menu = $ID['menu']; +echo "\n"; +echo "\n"; +echo "\n"; +echo "
\n"; + +$keysToRowIndex = array( + 'hist.exp_time' => 6, + 'hist.airmass' => 7, + 'hist.fwhm_major_psf' => 8, + 'hist.fwhm_minor_psf' => 9, + 'hist.fwhm_uq'=> 10, + 'hist.fwhm_lq' => 11, + 'hist.fwhm_major_moments' => 12, + 'hist.fwhm_minor_moments' => 13, + 'hist.fwhm_m2_moment' => 14, + 'hist.fwhm_m3_moment' => 15, + 'hist.fwhm_m4_moment' =>16); +$handles = array(); +$filenames = array(); +foreach ($keysToRowIndex as $i => $value) { + if (get_value_from_key($i, TRUE) == "on") { + $filenames[$i] = tempnam("/tmp/serge", "histodat."); + if ($debug) { + echo "Using $filenames[$i] for $i\n"; + } + $handles[$i] = fopen($filenames[$i], "w"); + } +} +// ** TAIL CODE ** +while ($qry->fetchInto($row)) { + foreach ($keysToRowIndex as $i => $value) { + if (get_value_from_key($i, TRUE) == "on") { + fwrite($handles[$i], + sprintf("%.3f\n", $row[$value])); + } + } +} +foreach ($keysToRowIndex as $i => $value) { + if ($handles[$i] != NULL) { + fclose($handles[$i]); + chmod($filenames[$i], 0777); + } +} + +if ($debug) { + $show = get_value_from_key('hist.exp_time', TRUE); + $show .= " && ".get_value_from_key('hist.airmass', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_major_psf', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_minor_psf', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_uq', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_lq', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_major_moments', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_minor_moments', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_m2_moment', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_m3_moment', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_m4_moment', TRUE); + echo "==> ($show)
\n"; +} + +//Get environment +$PATH = getenv("PATH"); +putenv("PATH=$BINDIR:$PATH"); +$LD_LIBRARY_PATH = getenv("LD_LIBRARY_PATH"); +putenv("LD_LIBRARY_PATH=$LIBDIR:$LD_LIBRARY_PATH"); + +echo "
\n"; +foreach ($filenames as $i => $value) { + $outfile = $value.".png"; + $deltaValue = get_value_from_key($i."_delta", TRUE); + $binValue = get_value_from_key($i."_count", TRUE); + if ($binValue == "") { + $binValue = "-1"; + if ($deltaValue == "") { + $binValue = "100"; + $deltaValue = "-1"; + } + } else { + if ($deltaValue != "") { + echo "Warning! Both 'Bins number' and 'delta value' are specified => 'delta value' will be ignored
\n"; + $deltaValue = "-1"; + } else { + $deltaValue = "-1"; + } + } + $title = preg_replace ('|hist\.|', '', $i); + $title = preg_replace ('|_|', '-', $title); + + echo "Showing histogram for [$title] / delta = $deltaValue / bins = $binValue
\n"; + exec("build_histogram.dvo $value $outfile $deltaValue $binValue $title", + $output, $status); + if ($debug) { + echo "    Status = $status
\n"; + for ($i = 0; $i < count($output); $i++) { + echo "    output $i: $output[$i]
"; + } + } + $blah=count($output); + if ($debug) { + echo "    end: $output / $blah
\n"; + } + echo "

\n"; + unset($output); + $filesToRemove[$value] = ''; +} + +echo " WHERE: $WHERE

\n"; +echo " SQL: $sql

\n"; + +if ($debug) { + echo "Cleaning
\n"; +} +foreach ($filesToRemove as $index => $value) { + if ($debug) { + echo "Deleting $index
\n"; + } + unlink($index); +} +if ($debug) { + echo "Cleaning end
\n"; +} + +menu_end(); + +?> Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.imfiles.dat (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.imfiles.dat (revision 28439) @@ -22,3 +22,4 @@ menutop | menutop | link | simple plot - cam | simplePlotcam.php menutop | menutop | link | czartool | czartool_labels.php +menutop | menutop | link | histogram | histogram.php Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.php =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.php (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.php (revision 28439) @@ -564,6 +564,41 @@ } +// +function button_restrict_checkbox ($key, $line) { + $htmlkey = preg_replace ('|\.|', '_', $key); + if ($_SERVER[REQUEST_METHOD] == 'GET') { + $value = $_GET[$htmlkey]; + } else { + $value = $_POST[$htmlkey]; + } + if ($value != "") { + if ($line) { + $line = $line . "&$htmlkey=$value"; + } else { + $line = "$htmlkey=$value"; + } + } + return $line; +} + +// +function button_restrict_radio ($key, $line) { + $htmlkey = preg_replace ('|\.|', '_', $key); + if ($_SERVER[REQUEST_METHOD] == 'GET') { + $value = $_GET[$htmlkey]; + } else { + $value = $_POST[$htmlkey]; + } + if ($value != "") { + if ($line) { + $line = $line . "&$htmlkey=$value"; + } else { + $line = "$htmlkey=$value"; + } + } + return $line; +} + function write_header_cell ($class, $name) { - echo "$name\n"; } @@ -637,6 +672,45 @@ if ($value != "") { echo "value=\"$value\">"; - } + } else { + echo ">"; + } echo "\n"; +} + +// checkbox +function write_query_checkbox ($key, $comment) { + $htmlkey = preg_replace ('|\.|', '_', $key); + + if ($_SERVER[REQUEST_METHOD] == 'GET') { + $value = $_GET[$htmlkey]; + } else { + $value = $_POST[$htmlkey]; + } + echo " "; + } else { + echo ">"; + } + echo "$comment ($htmlkey/$value)\n"; +} + +// checkbox +function write_query_radio ($value, $key) { + $htmlkey = preg_replace ('| |', '_', $key); + $htmlvalue = preg_replace ('|\.|', '_', $value); + + if ($_SERVER[REQUEST_METHOD] == 'GET') { + $value = $_GET[$htmlkey]; + } else { + $value = $_POST[$htmlkey]; + } + if ($htmlvalue == get_value_from_key($htmlkey, TRUE)) { + $checked = "checked"; + } else { + $checked = ""; + } + echo " "; + echo "$key ($htmlkey/$htmlvalue)\n"; } @@ -778,3 +852,22 @@ // $myPage = $_SERVER[SCRIPT_NAME] . "?pass=$pass"; +////////////////////////////////////////////////////////////////////////// +// Return the value associated to the key (GET or POST method) +// If convertKeyToHtmlkey is true, the key is converted to a +// so-called HTML, i.e. the key where '.' are replaced by '_' +// (e.g. the conversion of 'f.o.o.b_a_r' is 'f_o_o_b_a_r'. +// +function get_value_from_key ($key, $convertKeyToHtmlkey) { + if ($convertKeyToHtmlkey) { + $htmlkey = preg_replace ('|\.|', '_', $key); + } else { + $htmlkey = $key; + } + if ($_SERVER[REQUEST_METHOD] == 'GET') { + return $_GET[$htmlkey]; + } else { + return $_POST[$htmlkey]; + } +} + ?> Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/show_and_delete_image.php =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/raw/show_and_delete_image.php (revision 28439) +++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/show_and_delete_image.php (revision 28439) @@ -0,0 +1,18 @@ + Index: /branches/eam_branches/ipp-20100621/ippMonitor/scripts/build_histogram.dvo =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/scripts/build_histogram.dvo (revision 28439) +++ /branches/eam_branches/ipp-20100621/ippMonitor/scripts/build_histogram.dvo (revision 28439) @@ -0,0 +1,78 @@ +#!/usr/bin/env dvo + +$debug = 0 + +$bins = 100 +$delta = -1 + +macro build_histogram + myecho "Loading data from = $1" + data $1 + read v 1 + minimum v + maximum v + myecho "minvalue = $minvalue" + myecho "maxvalue = $maxvalue" + + if ($bins >= 2) + $delta = ($maxvalue-$minvalue)/$bins + myecho "delta = $delta" + else + if ($delta <= 0) + $bins = 100 + $delta = ($maxvalue-$minvalue)/$bins + myecho "delta = $delta" + end + end + + histo v vhist $minvalue $maxvalue $delta -range dx + myecho "bins = dx[]" + myecho " Setting limits" + limits dx vhist + plot dx vhist -x 1 + clear + myecho " Showing box" + box + myecho " Plotting" + plot dx vhist -x 1 -c blue + $binsCount = dx[] + myecho " Labeling" + myecho " -x $3" + myecho " -y Occurrences " + myecho " +x 'Histogram of $3 " + myecho " interval = $minvalue : $maxvalue " + myecho " / $binsCount bins" + myecho " / bin width: $delta)'" + labels -x $3 + labels -y Occurrences + labels +x "Histogram of $3 / interval = $minvalue : $maxvalue / $binsCount bins / bin width: $delta" + + myecho "Saving graphics to [$2]" + png -name $2 +end + +if ($SCRIPT) + $KAPA = kapa -noX + resize 1000 1000 + if ($argv:n != 5) + echo "USAGE: build_histogram (input) (output) (delta) (bins) (x-axis label)" + echo " If (bins) >= 2" + echo " build histogram with (bins) bins ((delta) is ignored)" + echo " Otherwise" + echo " if delta > 0 + echo " build histogram with (delta)-wide bins" + echo " otherwise" + echo " build histogram with 100 bins" + exit 1 + end + + echo "Loading dependencies" + $helpersFilename = `which helpers.dvo` + input $helpersFilename + echo "Dependencies loaded" + + $bins = $argv:3 + 1 + $delta = $argv:2 + build_histogram $argv:0 $argv:1 $argv:4 + exit 0 +end Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl (revision 28439) @@ -17,4 +17,5 @@ my $missing_tools = 0; my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $dqstatstool = can_run('dqstatstool') or (warn "Can't find dqstatstool" and $missing_tools = 1); my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); @@ -28,22 +29,4 @@ exit($PS_EXIT_CONFIG_ERROR); } - -# my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000'); -# my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10', -# 'STS','SVS','SweetSpot','ThreePi'); -# my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02', -# 'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06', -# 'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10', -# 'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0'); -# my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%', -# 'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%', -# 'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%', -# 'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%'); -# my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1, -# 'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1, -# 'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1, -# 'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0); -# my $retention_time = 9000; # days. - my $db; @@ -60,5 +43,5 @@ my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old); -my ( $check_detrends, $queue_detrends); +my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats); GetOptions( @@ -83,6 +66,8 @@ 'check_stacks' => \$check_stacks, 'queue_stacks' => \$queue_stacks, - 'check_detrends' => \$check_detrends, - 'queue_detrends' => \$queue_detrends, + 'check_detrends' => \$check_detrends, + 'queue_detrends' => \$queue_detrends, + 'check_dqstats' => \$check_dqstats, + 'queue_dqstats' => \$queue_dqstats, 'check_diffs' => \$check_diffs, 'queue_diffs' => \$queue_diffs, @@ -111,4 +96,6 @@ --check_detrends Confirm that detrend verify runs can be built. --queue_detrends Issue dettool commands to queue detrend verify runs. + --check_dqstats Confirm that dqstats tables can be built. + --queue_dqstats Issue dqstatstool commands to queue dqstat tables. --check_diffs Confirm that diffs can be done. --queue_diffs Issue difftool commands to queue diffs. @@ -121,6 +108,6 @@ ) unless defined $check_registration or defined $define_burntool or defined $queue_burntool or - defined $queue_chips or defined $queue_stacks or $queue_detrends or - defined $check_chips or defined $check_stacks or $check_detrends or + defined $queue_chips or defined $queue_stacks or $queue_detrends or $queue_dqstats or + defined $check_chips or defined $check_stacks or $check_detrends or $check_dqstats or defined $test_mode or defined $clean_old or defined $check_mode; @@ -132,4 +119,5 @@ my %object_list = (); my %comment_list= (); +my %cleanmods_list = (); my %stackable_list = (); my %reduction_class = (); @@ -144,4 +132,5 @@ my %clean_retention = (); my %noclean_list = (); +my %clean_alternate = (); # Grab the configuration data. my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; @@ -150,5 +139,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR); } @@ -169,4 +158,7 @@ $clean_retention{$this_mode} = ${ $mentry }{value}; } + elsif (${ $mentry }{name} eq 'ALTERNATE_CMD') { + $clean_alternate{$this_mode} = ${ $mentry }{value}; + } } } @@ -202,4 +194,9 @@ elsif (${ $tentry }{name} eq 'NOCLEAN') { $noclean_list{$this_target} = ${ $tentry }{value}; + } + else { + if (exists($clean_commands{ ${ $tentry }{name} })) { + $cleanmods_list{$this_target}{${ $tentry }{name} } = ${ $tentry }{value}; + } } } @@ -292,5 +289,17 @@ unless (defined($test_mode) || defined($check_mode)) { exit(0); } } -if (defined($check_detrends) || defined($test_mode)) { +if (defined($check_dqstats) || defined($test_mode)) { + $metadata_out{nsState} = 'CHECKDQSTATS'; + &execute_dqstats($date,"pretend"); + return_metadata($date); + unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($queue_dqstats) || defined($test_mode)) { + $metadata_out{nsState} = 'QUEUEDQSTATS'; + &execute_dqstats($date); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($check_detrends) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'CHECKDETRENDS'; &execute_detrends($date,"pretend"); @@ -300,5 +309,5 @@ if (defined($queue_detrends)) { $metadata_out{nsState} = 'QUEUEDETRENDS'; - &execute_detrends($date,"pretend"); + &execute_detrends($date); return_metadata($date); exit(0); @@ -613,6 +622,4 @@ } - - sub chip_queue { my $date = shift; @@ -660,4 +667,80 @@ } +# +# DQstats +################################################################################ + +sub construct_dqstats_cmd { + my $date = shift; + + my $select = "-dateobs_end ${date}T23:59:59 "; + + my $cmd = "$dqstatstool"; + $cmd .= " -simple -dbname $dbname -definebyquery "; + $cmd .= " $select "; + $cmd .= " -label %.nightlyscience "; + $cmd .= " -set_label dqstats.nightlyscience "; + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub pre_dqstats_queue { + my $date = shift; + +# my $db = init_gpc_db(); +# my $trunc_date = $date; $trunc_date =~ s/-//g; + +# my $where = " label LIKE '%.nightlyscience' AND data_group' "; +# my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new' OR state = 'cleaned') AND $where "; +# my $cam_sth = "SELECT * from camRun WHERE state = 'full' AND $where "; + +# my $chip_ref = $db->selectall_arrayref( $chip_sth ); +# my $cam_ref = $db->selectall_arrayref( $cam_sth ); + + my $command = construct_dqstats_cmd($date) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dqstatstool: $error_code",1,1,,$date, $PS_EXIT_SYS_ERROR); + } + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + + return($#input_exposures + 1,1,1); # $#{ $chip_ref } + 1, $#{ $cam_ref } + 1); +} + +sub dqstats_queue { + my $date = shift; + + my $command = construct_dqstatstool_cmd($date); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dqstatstool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + return(0); + +} +sub execute_dqstats { + my $date = shift; + my $pretend = shift; + my ($Nexposures,$Nchips,$Ncams) = pre_dqstats_queue($date); + if ($Nexposures == 0) { + print STDERR "execute_dqstats: No exposures on date $date.\n"; + } + elsif ($Ncams != $Nchips) { + print STDERR "execute_dqstats: Not done processing data through camera stage.\n"; + } + else { + unless(defined($pretend)) { + dqstats_queue($date); + } + } +} # # Detrend verification @@ -735,6 +818,6 @@ $metadata_out{nsState} = 'DETREND_QUEUED'; return(0); -} - + +} sub execute_detrends { my $date = shift; @@ -754,4 +837,7 @@ if ($exposures == 0) { $metadata_out{nsState} = 'DETREND_DROP'; + } + if (($metadata_out{nsState} eq 'CHECKDETRENDS') && ($exposures > 0)) { + $metadata_out{nsState} eq 'QUEUE_DETRENDS'; } } @@ -914,5 +1000,14 @@ my $command = $clean_commands{$mode}; - my $retention_time = $clean_retention{$mode}; + my $retention_time; + if (exists($cleanmods_list{$target}{$mode})) { + $retention_time = $cleanmods_list{$target}{$mode}; + } + else { + $retention_time = $clean_retention{$mode}; + } + if ($retention_time <= 0) { + return("no clean","true"); + } my ($year,$month,$day) = split /-/,$date; @@ -920,4 +1015,5 @@ hour => 0, minute => 0, second => 0, nanosecond => 0, time_zone => 'Pacific/Honolulu'); + $dt->subtract(days => $retention_time); my $cleaning_date = $dt->ymd; @@ -925,5 +1021,10 @@ my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); my $args = $command; - $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; + if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) { + $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -time_stamp_end $cleaning_date "; + } + else { + $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; + } if ($debug == 1) { $args .= ' -pretend '; @@ -936,11 +1037,10 @@ my $pretend = shift; - foreach my $mode (keys (%clean_commands)) { - foreach my $target (@target_list) { - if (exists($noclean_list{$target})) { + foreach my $mode (sort (keys (%clean_commands))) { + if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) { + my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode); + if ($cleaning_date eq 'no clean') { next; } - my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); - print STDERR "$command\n"; if (!(defined($pretend) || $debug == 1)) { @@ -951,4 +1051,25 @@ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + } + } + else { + foreach my $target (@target_list) { + if (exists($noclean_list{$target})) { + next; + } + my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); + if ($cleaning_date eq 'no clean') { + next; + } + print STDERR "$command\n"; + if (!(defined($pretend) || $debug == 1)) { +# print STDERR "BEAR IS DRIVING!?\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } } } Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/diffphot.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/diffphot.pl (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/diffphot.pl (revision 28439) @@ -70,6 +70,8 @@ # Get input components my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -my ($inputPath, $camera); # Input path and camera -my $bothways; # Diff was done both ways? +my $inputPath; # Input path +my $camera; # Camera name +my $bothways; # Diff was done both ways? +my $magicked; # Magic status { my $command = "$diffphottool -input -diff_phot_id $diff_phot_id -skycell_id $skycell_id"; @@ -89,4 +91,5 @@ $camera = $file->{camera}; $bothways = $file->{bothways}; + $magicked = $file->{magicked}; } @@ -252,4 +255,5 @@ my $command = "$diffphottool -diff_phot_id $diff_phot_id -skycell_id $skycell_id"; $command .= " -done -path_base $outroot"; + $command .= " -magicked $magicked" if $magicked; $command .= " -quality $quality" if defined $quality; $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); Index: /branches/eam_branches/ipp-20100621/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/eam_branches/ipp-20100621/ippTasks/ipphosts.mhpcc.config (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippTasks/ipphosts.mhpcc.config (revision 28439) @@ -1,103 +1,142 @@ ipphosts MULTI +# In this file, we assign all storage hosts to chip and skycells any +# hosts which are having problems are removed as active volumes from +# nebulous. data targeted to those machines are simply re-distributed +# randomly. the only exception is ipp004, which is the DVO db host + +# NOTE: wave 1 machines have 10TB; wave 2 & 3 machines have 20TB +# for balance, use 1 host per 10 TB in the partition ipphosts METADATA camera STR skycell - count S32 46 - sky00 STR ipp006 - sky01 STR ipp007 - sky02 STR ipp008 - sky03 STR ipp009 - sky04 STR ipp010 - sky05 STR ipp011 - sky06 STR ipp012 - sky07 STR ipp013 -# sky08 STR ipp014 - sky08 STR ipp047 - sky09 STR ipp015 -# sky10 STR ipp016 - sky10 STR ipp042 - sky11 STR ipp017 - sky12 STR ipp018 - sky13 STR ipp019 - sky14 STR ipp020 - sky15 STR ipp021 - sky16 STR ipp023 - sky17 STR ipp024 - sky18 STR ipp025 - sky19 STR ipp026 - sky20 STR ipp027 - sky21 STR ipp028 - sky22 STR ipp029 - sky23 STR ipp030 - sky24 STR ipp031 - sky25 STR ipp032 - sky26 STR ipp033 - sky27 STR ipp034 - sky28 STR ipp035 - sky29 STR ipp036 - sky30 STR ipp038 - sky31 STR ipp039 - sky32 STR ipp040 - sky33 STR ipp041 - sky34 STR ipp042 - sky35 STR ipp043 - sky36 STR ipp044 - sky37 STR ipp045 - sky38 STR ipp046 - sky39 STR ipp047 - sky40 STR ipp048 - sky41 STR ipp049 - sky42 STR ipp050 - sky43 STR ipp051 - sky44 STR ipp052 - sky45 STR ipp053 + count S32 79 + sky00 STR ipp005 + sky01 STR ipp006 + sky02 STR ipp007 + sky03 STR ipp008 + sky04 STR ipp009 + sky05 STR ipp010 + sky06 STR ipp011 + sky07 STR ipp012 + sky08 STR ipp013 + sky09 STR ipp014 + sky10 STR ipp015 + sky11 STR ipp016 + sky12 STR ipp017 + sky13 STR ipp018 + sky14 STR ipp019 + sky15 STR ipp020 + sky16 STR ipp021 + sky17 STR ipp023 + sky18 STR ipp024 + sky19 STR ipp025 + sky20 STR ipp026 + sky21 STR ipp027 + sky22 STR ipp028 + sky23 STR ipp029 + sky24 STR ipp030 + sky25 STR ipp031 + sky26 STR ipp032 + sky27 STR ipp033 + sky28 STR ipp034 + sky29 STR ipp035 + sky30 STR ipp036 + sky31 STR ipp037 + sky32 STR ipp038 + sky33 STR ipp039 + sky34 STR ipp040 + sky35 STR ipp041 + sky36 STR ipp042 + sky37 STR ipp043 + sky38 STR ipp044 + sky39 STR ipp045 + sky40 STR ipp046 + sky41 STR ipp047 + sky42 STR ipp048 + sky43 STR ipp049 + sky44 STR ipp050 + sky45 STR ipp051 + sky46 STR ipp052 + sky47 STR ipp053 + sky48 STR ipp023 + sky49 STR ipp024 + sky50 STR ipp025 + sky51 STR ipp026 + sky52 STR ipp027 + sky53 STR ipp028 + sky54 STR ipp029 + sky55 STR ipp030 + sky56 STR ipp031 + sky57 STR ipp032 + sky58 STR ipp033 + sky59 STR ipp034 + sky60 STR ipp035 + sky61 STR ipp036 + sky62 STR ipp037 + sky63 STR ipp038 + sky64 STR ipp039 + sky65 STR ipp040 + sky66 STR ipp041 + sky67 STR ipp042 + sky68 STR ipp043 + sky69 STR ipp044 + sky70 STR ipp045 + sky71 STR ipp046 + sky72 STR ipp047 + sky73 STR ipp048 + sky74 STR ipp049 + sky75 STR ipp050 + sky76 STR ipp051 + sky77 STR ipp052 + sky78 STR ipp053 END + +# we have 18 10TB nodes and 30 20TB nodes (ipp037 is used for dvo) +# 1 x W1 + 1 x W2 + 1 x W3 + 12 more nodes... ipphosts METADATA camera STR GPC1 -# XY01 STR ipp014 -# XY02 STR ipp014 - XY01 STR ipp047 - XY02 STR ipp047 - XY03 STR ipp038 + XY01 STR ipp037 + XY02 STR ipp005 + XY03 STR ipp006 XY04 STR ipp038 - XY05 STR ipp023 + XY05 STR ipp007 XY06 STR ipp023 - XY10 STR ipp039 + XY10 STR ipp008 XY11 STR ipp039 - XY12 STR ipp024 + XY12 STR ipp009 XY13 STR ipp024 - XY14 STR ipp040 + XY14 STR ipp010 XY15 STR ipp040 - XY16 STR ipp026 + XY16 STR ipp011 XY17 STR ipp026 - XY20 STR ipp041 + XY20 STR ipp012 XY21 STR ipp041 - XY22 STR ipp042 + XY22 STR ipp013 XY23 STR ipp042 - XY24 STR ipp043 + XY24 STR ipp014 XY25 STR ipp043 - XY26 STR ipp028 + XY26 STR ipp027 XY27 STR ipp028 - XY30 STR ipp044 + XY30 STR ipp016 XY31 STR ipp044 - XY32 STR ipp029 + XY32 STR ipp017 XY33 STR ipp029 - XY34 STR ipp045 + XY34 STR ipp018 XY35 STR ipp045 - XY36 STR ipp030 + XY36 STR ipp019 XY37 STR ipp030 - XY40 STR ipp046 + XY40 STR ipp020 XY41 STR ipp046 - XY42 STR ipp031 + XY42 STR ipp021 XY43 STR ipp031 XY44 STR ipp047 @@ -129,5 +168,5 @@ XY72 STR ipp052 - XY73 STR ipp015 + XY73 STR ipp053 XY74 STR ipp015 XY75 STR ipp025 @@ -138,77 +177,75 @@ camera STR gpc1 -# ota01 STR ipp014 -# ota02 STR ipp014 - ota01 STR ipp047 - ota02 STR ipp047 - ota03 STR ipp038 + ota01 STR ipp037 + ota02 STR ipp005 + ota03 STR ipp006 ota04 STR ipp038 - - ota05 STR ipp023 + + ota05 STR ipp007 ota06 STR ipp023 - ota10 STR ipp039 + ota10 STR ipp008 ota11 STR ipp039 - - ota12 STR ipp024 + + ota12 STR ipp009 ota13 STR ipp024 - ota14 STR ipp040 + ota14 STR ipp010 ota15 STR ipp040 - - ota16 STR ipp026 + + ota16 STR ipp011 ota17 STR ipp026 - ota20 STR ipp041 + ota20 STR ipp012 ota21 STR ipp041 - - ota22 STR ipp042 + + ota22 STR ipp013 ota23 STR ipp042 - ota24 STR ipp043 + ota24 STR ipp014 ota25 STR ipp043 - - ota26 STR ipp028 + + ota26 STR ipp027 ota27 STR ipp028 - ota30 STR ipp044 + ota30 STR ipp016 ota31 STR ipp044 - - ota32 STR ipp029 + + ota32 STR ipp017 ota33 STR ipp029 - ota34 STR ipp045 + ota34 STR ipp018 ota35 STR ipp045 - - ota36 STR ipp030 + + ota36 STR ipp019 ota37 STR ipp030 - ota40 STR ipp046 + ota40 STR ipp020 ota41 STR ipp046 - - ota42 STR ipp031 + + ota42 STR ipp021 ota43 STR ipp031 ota44 STR ipp047 ota45 STR ipp047 - + ota46 STR ipp032 ota47 STR ipp032 ota50 STR ipp048 ota51 STR ipp048 - + ota52 STR ipp033 ota53 STR ipp033 ota54 STR ipp049 ota55 STR ipp049 - + ota56 STR ipp034 ota57 STR ipp034 ota60 STR ipp050 ota61 STR ipp050 - + ota62 STR ipp035 ota63 STR ipp035 ota64 STR ipp051 ota65 STR ipp051 - + ota66 STR ipp036 ota67 STR ipp036 ota71 STR ipp052 ota72 STR ipp052 - - ota73 STR ipp015 + + ota73 STR ipp053 ota74 STR ipp015 ota75 STR ipp025 @@ -217,34 +254,35 @@ # this list is no longer used -ipphosts METADATA - camera STR distribution - count S32 26 - 0 STR ipp021 - 1 STR ipp023 - 2 STR ipp024 - 3 STR ipp026 - 4 STR ipp028 - 5 STR ipp029 - 6 STR ipp030 - 7 STR ipp031 - 8 STR ipp032 - 9 STR ipp033 - 10 STR ipp034 - 11 STR ipp035 - 12 STR ipp036 - 13 STR ipp038 - 14 STR ipp039 - 15 STR ipp040 - 16 STR ipp041 - 17 STR ipp043 - 18 STR ipp044 - 19 STR ipp045 - 20 STR ipp046 - 21 STR ipp047 - 22 STR ipp048 - 23 STR ipp050 - 24 STR ipp051 - 25 STR ipp052 -END +# XXX : delete if this does not cause trouble +## ipphosts METADATA +## camera STR distribution +## count S32 26 +## 0 STR ipp021 +## 1 STR ipp023 +## 2 STR ipp024 +## 3 STR ipp026 +## 4 STR ipp028 +## 5 STR ipp029 +## 6 STR ipp030 +## 7 STR ipp031 +## 8 STR ipp032 +## 9 STR ipp033 +## 10 STR ipp034 +## 11 STR ipp035 +## 12 STR ipp036 +## 13 STR ipp038 +## 14 STR ipp039 +## 15 STR ipp040 +## 16 STR ipp041 +## 17 STR ipp043 +## 18 STR ipp044 +## 19 STR ipp045 +## 20 STR ipp046 +## 21 STR ipp047 +## 22 STR ipp048 +## 23 STR ipp050 +## 24 STR ipp051 +## 25 STR ipp052 +## END ipphosts METADATA @@ -252,77 +290,75 @@ count S32 60 -# XY01 STR ipp014 -# XY02 STR ipp014 - XY01 STR ipp047 - XY02 STR ipp047 - XY03 STR ipp038 + XY01 STR ipp037 + XY02 STR ipp005 + XY03 STR ipp006 XY04 STR ipp038 - - XY05 STR ipp023 + + XY05 STR ipp007 XY06 STR ipp023 - XY10 STR ipp039 + XY10 STR ipp008 XY11 STR ipp039 - - XY12 STR ipp024 + + XY12 STR ipp009 XY13 STR ipp024 - XY14 STR ipp040 + XY14 STR ipp010 XY15 STR ipp040 - - XY16 STR ipp026 + + XY16 STR ipp011 XY17 STR ipp026 - XY20 STR ipp041 + XY20 STR ipp012 XY21 STR ipp041 - - XY22 STR ipp042 + + XY22 STR ipp013 XY23 STR ipp042 - XY24 STR ipp043 + XY24 STR ipp014 XY25 STR ipp043 - - XY26 STR ipp028 + + XY26 STR ipp027 XY27 STR ipp028 - XY30 STR ipp044 + XY30 STR ipp016 XY31 STR ipp044 - - XY32 STR ipp029 + + XY32 STR ipp017 XY33 STR ipp029 - XY34 STR ipp045 + XY34 STR ipp018 XY35 STR ipp045 - - XY36 STR ipp030 + + XY36 STR ipp019 XY37 STR ipp030 - XY40 STR ipp046 + XY40 STR ipp020 XY41 STR ipp046 - - XY42 STR ipp031 + + XY42 STR ipp021 XY43 STR ipp031 XY44 STR ipp047 XY45 STR ipp047 - + XY46 STR ipp032 XY47 STR ipp032 XY50 STR ipp048 XY51 STR ipp048 - + XY52 STR ipp033 XY53 STR ipp033 XY54 STR ipp049 XY55 STR ipp049 - + XY56 STR ipp034 XY57 STR ipp034 XY60 STR ipp050 XY61 STR ipp050 - + XY62 STR ipp035 XY63 STR ipp035 XY64 STR ipp051 XY65 STR ipp051 - + XY66 STR ipp036 XY67 STR ipp036 XY71 STR ipp052 XY72 STR ipp052 - - XY73 STR ipp015 + + XY73 STR ipp053 XY74 STR ipp015 XY75 STR ipp025 @@ -333,51 +369,82 @@ camera STR dist_skycell count S32 46 - sky00 STR ipp006 - sky01 STR ipp007 - sky02 STR ipp008 - sky03 STR ipp009 - sky04 STR ipp010 - sky05 STR ipp011 - sky06 STR ipp012 - sky07 STR ipp013 -# sky08 STR ipp014 - sky08 STR ipp047 - sky09 STR ipp015 -# sky10 STR ipp016 - sky10 STR ipp042 - sky11 STR ipp017 - sky12 STR ipp018 - sky13 STR ipp019 - sky14 STR ipp020 - sky15 STR ipp021 - sky16 STR ipp023 - sky17 STR ipp024 - sky18 STR ipp025 - sky19 STR ipp026 - sky20 STR ipp027 - sky21 STR ipp028 - sky22 STR ipp029 - sky23 STR ipp030 - sky24 STR ipp031 - sky25 STR ipp032 - sky26 STR ipp033 - sky27 STR ipp034 - sky28 STR ipp035 - sky29 STR ipp036 - sky30 STR ipp038 - sky31 STR ipp039 - sky32 STR ipp040 - sky33 STR ipp041 - sky34 STR ipp042 - sky35 STR ipp043 - sky36 STR ipp044 - sky37 STR ipp045 - sky38 STR ipp046 - sky39 STR ipp047 - sky40 STR ipp048 - sky41 STR ipp049 - sky42 STR ipp050 - sky43 STR ipp051 - sky44 STR ipp052 - sky45 STR ipp053 + sky00 STR ipp005 + sky01 STR ipp006 + sky02 STR ipp007 + sky03 STR ipp008 + sky04 STR ipp009 + sky05 STR ipp010 + sky06 STR ipp011 + sky07 STR ipp012 + sky08 STR ipp013 + sky09 STR ipp014 + sky10 STR ipp015 + sky11 STR ipp016 + sky12 STR ipp017 + sky13 STR ipp018 + sky14 STR ipp019 + sky15 STR ipp020 + sky16 STR ipp021 + sky17 STR ipp023 + sky18 STR ipp024 + sky19 STR ipp025 + sky20 STR ipp026 + sky21 STR ipp027 + sky22 STR ipp028 + sky23 STR ipp029 + sky24 STR ipp030 + sky25 STR ipp031 + sky26 STR ipp032 + sky27 STR ipp033 + sky28 STR ipp034 + sky29 STR ipp035 + sky30 STR ipp036 + sky31 STR ipp037 + sky32 STR ipp038 + sky33 STR ipp039 + sky34 STR ipp040 + sky35 STR ipp041 + sky36 STR ipp042 + sky37 STR ipp043 + sky38 STR ipp044 + sky39 STR ipp045 + sky40 STR ipp046 + sky41 STR ipp047 + sky42 STR ipp048 + sky43 STR ipp049 + sky44 STR ipp050 + sky45 STR ipp051 + sky46 STR ipp052 + sky47 STR ipp053 + sky48 STR ipp023 + sky49 STR ipp024 + sky50 STR ipp025 + sky51 STR ipp026 + sky52 STR ipp027 + sky53 STR ipp028 + sky54 STR ipp029 + sky55 STR ipp030 + sky56 STR ipp031 + sky57 STR ipp032 + sky58 STR ipp033 + sky59 STR ipp034 + sky60 STR ipp035 + sky61 STR ipp036 + sky62 STR ipp037 + sky63 STR ipp038 + sky64 STR ipp039 + sky65 STR ipp040 + sky66 STR ipp041 + sky67 STR ipp042 + sky68 STR ipp043 + sky69 STR ipp044 + sky70 STR ipp045 + sky71 STR ipp046 + sky72 STR ipp047 + sky73 STR ipp048 + sky74 STR ipp049 + sky75 STR ipp050 + sky76 STR ipp051 + sky77 STR ipp052 + sky78 STR ipp053 END Index: /branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro =================================================================== --- /branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro (revision 28439) @@ -6,4 +6,5 @@ ns.initday.on ns.detrends.off + ns.dqstats.on ns.registration.on ns.burntool.on @@ -51,4 +52,10 @@ end +macro ns.dqstats.on + task ns.dqstats.load + active true + end +end + macro ns.registration.on task ns.registration.load @@ -92,4 +99,10 @@ macro ns.detrends.off task ns.detrends.load + active false + end +end + +macro ns.dqstats.off + task ns.dqstats.load active false end @@ -227,4 +240,40 @@ task.exit 0 # nothign to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Queue dqstats runs +# +task ns.dqstats.load + host local + periods -poll 3600 + periods -exec $LOADEXEC + periods -timeout 30 + trange 22:00:00 23:59:59 + trange 00:00:00 10:00:00 + npending 1 + + task.exec + stdout $LOGDIR/ns.dqstats.log + stderr $LOGDIR/ns.dqstats.log + $today = `date +%Y-%m-%d` + + command automate_stacks.pl --queue_dqstats --date $today + end + + task.exit 0 + # nothing to do here end # locked list Index: /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsBatchDetection.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsBatchDetection.c (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsBatchDetection.c (revision 28439) @@ -13,14 +13,8 @@ #include "fitsio.h" -// Gets flux from magnitude +// Gets uncalibrated instrumental flux from magnitude static __inline bool ippToPsps_getFlux(const float zeroPoint, const float exposureTime, const float magnitude, float* flux) { - // use uncalibrated instrumental flux *flux = powf(10.0, -0.4*magnitude) / exposureTime; - - // use calibrated flux in Janskys, where 3631 Jy is the zero point flux for the filter: constant over all filters because we're using AB magnitudes - //float flux = 3631 * powf(10.0, -0.4*(magnitude + zeroPoint)) / exposureTime; - - // printf("mag=%f\texpTime=%f\tzeroPoint=%f\tflux=%f\n", magnitude, exposureTime, zeroPoint, flux); return (!isfinite(*flux) || *flux < 0.000001) ? false : true; } @@ -104,5 +98,5 @@ int maxDvoDetId = -1; int numDvoDetections = -1; - Image *pImage = NULL; + Image *image = NULL; // stuff for detections table @@ -157,4 +151,5 @@ long maxObjID = LONG_MIN; long minObjID = LONG_MAX; + uint64_t imageFlags; short nOta = 0; long i; @@ -163,4 +158,5 @@ uint32_t numInvalidFlux; long numDetectionsOut; + long totalDetectionsOut = 0; long removeList[MAXDETECT]; long thisObjId; @@ -203,5 +199,5 @@ // access DVO database - skylist = dvoSkyListByExternID(this->dvoConfig, sourceId, imageId, &pImage); + skylist = dvoSkyListByExternID(this->dvoConfig, sourceId, imageId, &image); if (skylist == NULL) { psError(PS_ERR_IO, false, @@ -212,9 +208,9 @@ // create unique int from 'frameID' (aka exposure ID) and ccd number - pspsImageId = (this->expId*100) + pImage->ccdnum; + pspsImageId = (this->expId*100) + image->ccdnum; // now get DVO detections dvoDetections = NULL; - numDvoDetections = dvoGetDetections(skylist, pImage->imageID, &dvoDetections, &maxDvoDetId); + numDvoDetections = dvoGetDetections(skylist, image->imageID, &dvoDetections, &maxDvoDetId); // TODO check nDet < MAXDETECT @@ -224,8 +220,9 @@ psfFwhm = (fwhmMaj+fwhmMin)/2; momentFwhm = (momentMaj+momentMin)/2; + imageFlags = (uint64_t)image->flags; fits_write_col(this->fitsOut, TLONGLONG, IMAGEMETA_IMAGEID, 1, 1, 1, &pspsImageId, &status); fits_write_col(this->fitsOut, TLONG, IMAGEMETA_FRAMEID, 1, 1, 1, &this->expId, &status); - fits_write_col(this->fitsOut, TSHORT, IMAGEMETA_CCDID, 1, 1, 1, &pImage->ccdnum, &status); - fits_write_col(this->fitsOut, TLONG, IMAGEMETA_PHOTOCALID, 1, 1, 1, &pImage->photcode, &status); + fits_write_col(this->fitsOut, TSHORT, IMAGEMETA_CCDID, 1, 1, 1, &image->ccdnum, &status); + fits_write_col(this->fitsOut, TLONG, IMAGEMETA_PHOTOCALID, 1, 1, 1, &image->photcode, &status); fits_write_col(this->fitsOut, TBYTE, IMAGEMETA_FILTERID, 1, 1, 1, &filterID, &status); fits_write_col(this->fitsOut, TFLOAT, IMAGEMETA_PHOTOSCAT, 1, 1, 1, &zptObs, &status); @@ -233,4 +230,5 @@ fits_write_col(this->fitsOut, TFLOAT, IMAGEMETA_MOMENTFWHM, 1, 1, 1, &momentFwhm, &status); fits_write_col(this->fitsOut, TFLOAT, IMAGEMETA_PHOTOZERO, 1, 1, 1, &zptObs, &status); + fits_write_col(this->fitsOut, TLONGLONG, IMAGEMETA_QAFLAGS, 1, 1, 1, &imageFlags, &status); // now move BACK to detections table in smf @@ -365,8 +363,12 @@ extensionName, nDet, numDetectionsOut, unmatched, numOfDuplicates, numInvalidFlux); + totalDetectionsOut += numDetectionsOut; + if (dvoDetections!= NULL) dvoFree(dvoDetections); SkyListFree(skylist); } } + + psLogMsg("ippToPsps", PS_LOG_INFO, "Total detections for this exposure = %ld\n", totalDetectionsOut); for (uint32_t i=0; iresultsFile, "%ld\n", maxObjID) < 1 ) psError(PS_ERR_IO, false, "Unable to write max Object ID to'%s'\n", this->resultsPath); + if (fprintf(this->resultsFile, "%ld\n", totalDetectionsOut) < 1 ) + psError(PS_ERR_IO, false, "Unable to write totalDetectionsOut to'%s'\n", this->resultsPath); } Index: /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_input.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_input.sql (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_input.sql (revision 28439) @@ -2,4 +2,5 @@ diffPhotRun.*, diffSkyfile.path_base, + diffSkyfile.magicked, diffRun.bothways, rawExp.camera Index: /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_pending.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_pending.sql (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_pending.sql (revision 28439) @@ -9,4 +9,5 @@ WHERE diffPhotSkyfile.skycell_id IS NULL AND diffRun.state = 'full' + AND diffSkyfile.magicked >= 0 AND diffSkyfile.fault = 0 AND diffSkyfile.quality = 0 Index: /branches/eam_branches/ipp-20100621/ippTools/share/pubtool_definerun.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/pubtool_definerun.sql (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippTools/share/pubtool_definerun.sql (revision 28439) @@ -56,5 +56,5 @@ AND publishClient.active = 1 AND diffPhotRun.state IN ('full', 'cleaned', 'goto_cleaned') - AND (diffRun.diff_mode = 4 OR publishClient.magicked = 0) -- diffPhotRun doesn't respect magic + AND (diffPhotRun.magicked != 0 OR diffRun.diff_mode = 4 OR publishClient.magicked = 0) -- WHERE hook %s ) AS publishToDo Index: /branches/eam_branches/ipp-20100621/ippTools/share/pubtool_pending.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/pubtool_pending.sql (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippTools/share/pubtool_pending.sql (revision 28439) @@ -74,5 +74,5 @@ AND publishRun.state = 'new' AND diffPhotRun.state IN ('full', 'cleaned', 'goto_cleaned') - AND (diffRun.diff_mode = 4 OR publishClient.magicked = 0) -- diffPhotRun doesn't respect magic + AND (diffPhotRun.magicked != 0 OR diffRun.diff_mode = 4 OR publishClient.magicked = 0) -- WHERE hook %s ) AS publishToDo Index: /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_create_tables.sql (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_create_tables.sql (revision 28439) @@ -1755,4 +1755,5 @@ quality SMALLINT NOT NULL, -- bad quality flag software_ver VARCHAR(16), -- software version + magicked BIGINT NOT NULL DEFAULT 0, -- magic mask applied PRIMARY KEY(diff_phot_id, skycell_id), KEY(fault), Index: /branches/eam_branches/ipp-20100621/ippTools/src/diffphottool.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/diffphottool.c (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippTools/src/diffphottool.c (revision 28439) @@ -371,4 +371,5 @@ PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); psString version = pxMergeCodeVersions(ver_pslib, ver_psmodules); @@ -377,5 +378,5 @@ if (!diffPhotSkyfileInsert(config->dbh, diff_phot_id, skycell_id, path_base, dtime_script, hostname, - fault, quality, version)) { + fault, quality, version, magicked)) { psError(psErrorCodeLast(), false, "database error"); return false; Index: /branches/eam_branches/ipp-20100621/ippTools/src/diffphottoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/diffphottoolConfig.c (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippTools/src/diffphottoolConfig.c (revision 28439) @@ -100,4 +100,5 @@ psMetadataAddStr(doneArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); psMetadataAddStr(doneArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddS64(doneArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); // -advance Index: /branches/eam_branches/ipp-20100621/ippTools/src/dqstatstool.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/dqstatstool.c (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippTools/src/dqstatstool.c (revision 28439) @@ -85,5 +85,5 @@ if (label) { - PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "LIKE"); // define using cam label } // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE @@ -654,5 +654,15 @@ psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); return(false); - } + } + psString filter = psMetadataLookupStr(&status,rule,"FILTER"); + if (!status) { + status = true; + } + if (filter) { + psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER"); + if (strcmp(filter,imfilter) != 0) { + continue; + } + } // Not happy with this being set to a F32. Can this ever be something else? psF32 value = psMetadataLookupF32(&status,tableRow,colname); Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/dqStatsTool.config =================================================================== --- /branches/eam_branches/ipp-20100621/ippconfig/recipes/dqStatsTool.config (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/dqStatsTool.config (revision 28439) @@ -7,10 +7,46 @@ COLNAME STR FWHM_MAJOR MINIMUM F32 0.0 - MAXIMUM F32 5.0 + MAXIMUM F32 9.3 # 90-percentile 2010-02-14 through 2010-06-20 = 2.4" END RULE METADATA COLNAME STR FWHM_MINOR MINIMUM F32 0.0 - MAXIMUM F32 5.0 + MAXIMUM F32 8.44 # 90-percentile 2010-02-14 through 2010-06-20 = 2.18" + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR g.00000 + MINIMUM F32 24.3051 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR r.00000 + MINIMUM F32 24.5294 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR i.00000 + MINIMUM F32 24.4641 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR z.00000 + MINIMUM F32 24.1128 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR y.00000 + MINIMUM F32 23.1503 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR w.00000 + MINIMUM F32 25.8944 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. END # TYPE VAL COLNAME MINIMUM MAXIMUM Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/nightly_science.config =================================================================== --- /branches/eam_branches/ipp-20100621/ippconfig/recipes/nightly_science.config (revision 28438) +++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/nightly_science.config (revision 28439) @@ -6,4 +6,5 @@ COMMAND STR chiptool RETENTION_TIME U16 30 + ALTERNATE_CMD BOOL F END CLEAN_MODES METADATA @@ -11,10 +12,19 @@ COMMAND STR warptool RETENTION_TIME U16 7 -END -# CLEAN_MODES METADATA -# MODE STR DIFF -# COMMAND STR difftool -# RETENTION_TIME U16 30 -# END + ALTERNATE_CMD BOOL F +END +CLEAN_MODES METADATA + MODE STR DIFF + COMMAND STR difftool +# RETENTION_TIME U16 30 + RETENTION_TIME S16 -1 + ALTERNATE_CMD BOOL F +END +CLEAN_MODES METADATA + MODE STR DIST + COMMAND STR disttool + RETENTION_TIME S16 7 + ALTERNATE_CMD BOOL T +END FILTERS MULTI @@ -128,5 +138,6 @@ # OBJECT STR SS STACKABLE BOOL FALSE - NOCLEAN BOOL TRUE +# NOCLEAN BOOL TRUE + WARP S16 60 END TARGETS METADATA @@ -293,5 +304,5 @@ DETRENDS METADATA NAME STR DARK - DETTYPE STR DARK + DETTYPE STR DARK EXPTYPE STR dark REF_ID S64 299 @@ -303,5 +314,5 @@ DETRENDS METADATA NAME STR FLAT_G - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 300 @@ -313,5 +324,5 @@ DETRENDS METADATA NAME STR FLAT_R - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 301 @@ -323,5 +334,5 @@ DETRENDS METADATA NAME STR FLAT_I - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 302 @@ -333,5 +344,5 @@ DETRENDS METADATA NAME STR FLAT_Z - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 303 @@ -343,5 +354,5 @@ DETRENDS METADATA NAME STR FLAT_Y - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 304 @@ -353,5 +364,5 @@ DETRENDS METADATA NAME STR FLAT_W - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 305 Index: /branches/eam_branches/ipp-20100621/ppBackground/src/ppBackgroundArguments.c =================================================================== --- /branches/eam_branches/ipp-20100621/ppBackground/src/ppBackgroundArguments.c (revision 28438) +++ /branches/eam_branches/ipp-20100621/ppBackground/src/ppBackgroundArguments.c (revision 28439) @@ -30,5 +30,4 @@ fprintf(stderr, "\n"); psArgumentHelp(arguments); - psFree(arguments); psFree(data);