Index: /branches/eam_branches/ipp-20120905/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm
===================================================================
--- /branches/eam_branches/ipp-20120905/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 34465)
+++ /branches/eam_branches/ipp-20120905/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 34466)
@@ -25,4 +25,5 @@
$PSTAMP_SELECT_WEIGHT
$PSTAMP_SELECT_CMF
+ $PSTAMP_SELECT_SOURCES
$PSTAMP_SELECT_PSF
$PSTAMP_SELECT_BACKMDL
@@ -32,8 +33,8 @@
$PSTAMP_SELECT_UNCONV
$PSTAMP_RESTORE_BACKGROUND
+ $PSTAMP_USE_IMFILE_ID
+ $PSTAMP_NO_WAIT_FOR_UPDATE
$PSTAMP_REQUEST_UNCENSORED
$PSTAMP_REQUIRE_UNCENSORED
- $PSTAMP_USE_IMFILE_ID
- $PSTAMP_NO_WAIT_FOR_UPDATE
$PSTAMP_SUCCESS
$PSTAMP_FIRST_ERROR_CODE
@@ -59,22 +60,26 @@
our $PSTAMP_RANGE_IN_PIXELS = 2;
+# Definition of the bits in OPTION_MASK
our $PSTAMP_SELECT_IMAGE = 1;
our $PSTAMP_SELECT_MASK = 2;
our $PSTAMP_SELECT_VARIANCE = 4;
our $PSTAMP_SELECT_WEIGHT = 4;
+our $PSTAMP_SELECT_SOURCES = 8;
our $PSTAMP_SELECT_CMF = 8;
our $PSTAMP_SELECT_PSF = 16;
our $PSTAMP_SELECT_BACKMDL = 32;
our $PSTAMP_SELECT_JPEG = 64;
+# unused 128
+# unused 256
our $PSTAMP_SELECT_UNCOMPRESSED = 512;
-our $PSTAMP_SELECT_INVERSE = 1024;
-our $PSTAMP_SELECT_UNCONV = 2048;
-our $PSTAMP_RESTORE_BACKGROUND = 4096;
-
-our $PSTAMP_USE_IMFILE_ID = 16384;
-
+our $PSTAMP_SELECT_INVERSE = 1024;
+our $PSTAMP_SELECT_UNCONV = 2048;
+our $PSTAMP_RESTORE_BACKGROUND = 4096;
+# unused 8192
+
+our $PSTAMP_USE_IMFILE_ID = 16384;
our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768;
-our $PSTAMP_REQUEST_UNCENSORED = 1 << 16;
-our $PSTAMP_REQUIRE_UNCENSORED = 1 << 17;
+our $PSTAMP_REQUEST_UNCENSORED = 0x10000;
+our $PSTAMP_REQUIRE_UNCENSORED = 0x20000;
# job and result codes
@@ -162,10 +167,10 @@
my $fields_output;
{
- my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME";
+ my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USERNAME EMAIL";
my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
run(command => $command, verbose => $verbose);
$fields_output = join "", @$stdout_buf;
}
- my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
+ my (undef, $extname, $extver, $req_name, $action, $username, $email) = split " ", $fields_output;
# make sure the file contains what we are expecting
@@ -174,5 +179,5 @@
if !$extname or ($extname ne "PS1_PS_REQUEST");
die "REQ_NAME not found in $request_file_name" if (!$req_name);
- die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
+ die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "2" and $extver ne "1");
my %header;
@@ -180,4 +185,20 @@
$header{EXTVER} = $extver;
$header{EXTNAME} = $extname;
+ if ($extver > 1) {
+ $header{ACTION} = $action;
+ $header{USER} = $username;
+ $header{EMAIL} = $email;
+ } else {
+ $header{ACTION} = $action = "PROCESS";
+ $header{USERNAME} = 'null';
+ $header{EMAIL} = 'null';
+ }
+
+ if ($action eq "LIST") {
+ return (\%header, undef);
+ } elsif ($action ne "PROCESS") {
+ die "unexpected request ACTION found: $action in $request_file_name";
+ }
+
#
Index: /branches/eam_branches/ipp-20120905/ippMonitor/Makefile.in
===================================================================
--- /branches/eam_branches/ipp-20120905/ippMonitor/Makefile.in (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ippMonitor/Makefile.in (revision 34466)
@@ -14,4 +14,5 @@
PROGRAMS = \
$(DESTBIN)/skyplot.dvo \
+$(DESTBIN)/skycellplot.dvo \
$(DESTBIN)/czartool_getServerStatus.pl \
$(DESTBIN)/czartool_revert.pl \
@@ -57,4 +58,5 @@
$(DESTWWW)/simplePlotraw.php \
$(DESTWWW)/skyplot.php \
+$(DESTWWW)/skycellplot.php \
$(DESTWWW)/getimage.php \
$(DESTWWW)/ipp.czartool.dat \
@@ -211,4 +213,7 @@
$(DESTWWW)/simplePlotChipImage.php \
$(DESTWWW)/simplePlotCamImage.php \
+$(DESTWWW)/simplePlotStackImage.php \
+$(DESTWWW)/simplePlotStaticskyImage.php \
+$(DESTWWW)/simplePlotSkycalImage.php \
$(DESTWWW)/histogramBackgroundImage.php \
$(DESTWWW)/histogramCamProcessedExpImage.php \
Index: /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotCamImage.d
===================================================================
--- /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotCamImage.d (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotCamImage.d (revision 34466)
@@ -1,3 +1,3 @@
-TABLE camRun, chipRun, rawExp
+TABLE camProcessedExp, camRun, chipRun, rawExp
TITLE New sky plot - cam
FILE simplePlotCamImage.php
@@ -7,4 +7,5 @@
WHERE chipRun.exp_id = rawExp.exp_id
WHERE camRun.chip_id = chipRun.chip_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
# field size format name show link to extras
@@ -24,4 +25,5 @@
FIELD rawExp.bg, 5, %.2f, backgnd, restrict
FIELD rawExp.bg_stdev, 5, %.2f, stdev, restrict
+FIELD camProcessedExp.fwhm_major, 5, %.2f, FWHM_major, restrict
# What to plot
Index: /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotSkycalImage.d
===================================================================
--- /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotSkycalImage.d (revision 34466)
+++ /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotSkycalImage.d (revision 34466)
@@ -0,0 +1,29 @@
+TABLE skycalRun, skycalResult, stackRun, skycell
+TITLE New sky plot - skycal
+FILE simplePlotSkycalImage.php
+MENU ipp.plots.dat
+
+WHERE skycalRun.state != 'new'
+WHERE skycalRun.skycal_id = skycalResult.skycal_id
+WHERE skycalRun.stack_id = stackRun.stack_id
+WHERE stackRun.skycell_id = skycell.skycell_id
+WHERE stackRun.tess_id = skycell.tess_id
+
+# field size format name show link to extras
+FIELD skycalRun.skycal_id, 5, %d, Skycal ID, restrict
+FIELD staticskyRun.sky_id, 5, %d, Staticsky ID, restrict
+FIELD stackRun.stack_id, 5, %d, Stack ID, restrict
+FIELD skycalRun.label, 10, %s, Label, restrict
+FIELD skycell.radeg, 10, %.6f, RA (deg), restrict
+FIELD skycell.decdeg, 10, %.6f, DEC (deg), restrict
+FIELD stackRun.filter, 10, %s, FILTER, restrict
+FIELD skycalResult.sigma_ra, 5, %.2f, sigma_ra, restrict
+FIELD skycalResult.sigma_dec, 5, %.2f, sigma_dec, restrict
+FIELD skycalResult.zpt_obs, 5, %.2f, zpt_obs, restrict
+FIELD skycalResult.zpt_stdev, 5, %.2f, zpt_stdev, restrict
+FIELD skycalResult.quality, 5, %d, quality, restrict
+
+# What to plot
+TOPLOT skycell.radeg,skycell.decdeg,stackRun.filter
+PLOTTER skycellplot.php
+PLOTTITLE Sky Plot Skycal
Index: /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotStackImage.d
===================================================================
--- /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotStackImage.d (revision 34466)
+++ /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotStackImage.d (revision 34466)
@@ -0,0 +1,26 @@
+TABLE stackRun, stackSumSkyfile, skycell
+TITLE New sky plot - stack
+FILE simplePlotStackImage.php
+MENU ipp.plots.dat
+
+WHERE stackRun.state != 'new'
+WHERE stackRun.stack_id = stackSumSkyfile.stack_id
+WHERE stackRun.skycell_id = skycell.skycell_id
+WHERE stackRun.tess_id = skycell.tess_id
+
+# field size format name show link to extras
+FIELD stackRun.stack_id, 5, %d, Stack ID, restrict
+FIELD stackRun.label, 10, %s, Label, restrict
+FIELD skycell.radeg, 10, %.6f, RA (deg), restrict
+FIELD skycell.decdeg, 10, %.6f, DEC (deg), restrict
+FIELD stackRun.filter, 10, %s, FILTER, restrict
+FIELD stackSumSkyfile.bg, 5, %.2f, BG, restrict
+FIELD stackSumSkyfile.bg_stdev, 5, %.2f, BG STDEV, restrict
+FIELD stackSumSkyfile.good_frac, 5, %.2f, good_frac, restrict
+FIELD stackSumSkyfile.quality, 5, %d, quality, restrict
+
+
+# What to plot
+TOPLOT skycell.radeg,skycell.decdeg,stackRun.filter
+PLOTTER skycellplot.php
+PLOTTITLE Sky Plot Stack
Index: /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotStaticskyImage.d
===================================================================
--- /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotStaticskyImage.d (revision 34466)
+++ /branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotStaticskyImage.d (revision 34466)
@@ -0,0 +1,26 @@
+TABLE staticskyRun, staticskyResult, staticskyInput, stackRun, skycell
+TITLE New sky plot - staticsky
+FILE simplePlotStaticskyImage.php
+MENU ipp.plots.dat
+
+WHERE staticskyRun.state != 'new'
+WHERE staticskyRun.sky_id = staticskyResult.sky_id
+WHERE staticskyRun.sky_id = staticskyInput.sky_id
+WHERE staticskyInput.stack_id = stackRun.stack_id
+WHERE stackRun.skycell_id = skycell.skycell_id
+WHERE stackRun.tess_id = skycell.tess_id
+
+# field size format name show link to extras
+FIELD staticskyRun.sky_id, 5, %d, Staticsky ID, restrict
+FIELD stackRun.stack_id, 5, %d, Stack ID, restrict
+FIELD staticskyRun.label, 10, %s, Label, restrict
+FIELD skycell.radeg, 10, %.6f, RA (deg), restrict
+FIELD skycell.decdeg, 10, %.6f, DEC (deg), restrict
+FIELD stackRun.filter, 10, %s, FILTER, restrict
+FIELD staticskyResult.num_inputs, 5, %d, num_inputs, restrict
+FIELD staticskyResult.quality, 5, %d, quality, restrict
+
+# What to plot
+TOPLOT skycell.radeg,skycell.decdeg,stackRun.filter
+PLOTTER skycellplot.php
+PLOTTITLE Sky Plot Staticsky
Index: /branches/eam_branches/ipp-20120905/ippMonitor/raw/ipp.plots.dat
===================================================================
--- /branches/eam_branches/ipp-20120905/ippMonitor/raw/ipp.plots.dat (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ippMonitor/raw/ipp.plots.dat (revision 34466)
@@ -5,4 +5,7 @@
menulink | menuselect | link | Simple plot - chip | simplePlotChipImage.php
menulink | menuselect | link | Simple plot - cam | simplePlotCamImage.php
+menulink | menuselect | link | Simple plot - stack | simplePlotStackImage.php
+menulink | menuselect | link | Simple plot - staticsky | simplePlotStaticskyImage.php
+menulink | menuselect | link | Simple plot - skycal | simplePlotSkycalImage.php
menutop | menutop | plain | |
Index: /branches/eam_branches/ipp-20120905/ippMonitor/raw/simplePlotcam.php
===================================================================
--- /branches/eam_branches/ipp-20120905/ippMonitor/raw/simplePlotcam.php (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ippMonitor/raw/simplePlotcam.php (revision 34466)
@@ -28,5 +28,5 @@
// define restrictiosn to the queries
// ** TABLE RESTRICTIONS **
-$WHERE = "WHERE camRun.state != 'new' AND chipRun.exp_id = rawExp.exp_id AND camRun.chip_id = chipRun.chip_id";
+$WHERE = "WHERE camRun.state != 'new' AND chipRun.exp_id = rawExp.exp_id AND camRun.chip_id = chipRun.chip_id AND camProcessedExp.cam_id = camRun.cam_id";
$WHERE = check_restrict ('rawExp.exp_name', $WHERE, 'string', 1.0);
$WHERE = check_restrict ('rawExp.exp_id', $WHERE, 'string', 1.0);
@@ -62,4 +62,7 @@
$WHERE = check_restrict ('rawExp.bg_stdev', $WHERE, 'max', 1.0);
$WHERE = check_restrict ('rawExp.comment', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'max', 1.0);
$WHERE = check_ordering ('', $WHERE);
@@ -73,14 +76,14 @@
// get the result table count
-$sql = "SELECT count(*) FROM camRun, chipRun, rawExp $WHERE";
+$sql = "SELECT count(*) FROM camProcessedExp, camRun, chipRun, rawExp $WHERE";
$qry = $db->query($sql);
if (dberror($qry)) {
- echo "error reading camRun, chipRun, rawExp table count
\n";
+ echo "error reading camProcessedExp, camRun, chipRun, rawExp table count
\n";
echo "
count query : $sql
\n";
menu_end();
}
if (!$qry->fetchInto($row)) {
- echo "error reading camRun, chipRun, rawExp table count
\n";
+ echo "error reading camProcessedExp, camRun, chipRun, rawExp table count
\n";
echo "
count query : $sql
\n";
menu_end();
@@ -128,5 +131,7 @@
$buttonLink = button_restrict_max ('rawExp.bg_stdev', $buttonLink);
$buttonLink = button_restrict_string ('rawExp.comment', $buttonLink);
-
+$buttonLink = button_restrict_string ('camProcessedExp.fwhm_major', $buttonLink);
+$buttonLink = button_restrict_min ('camProcessedExp.fwhm_major', $buttonLink);
+$buttonLink = button_restrict_max ('camProcessedExp.fwhm_major', $buttonLink);
// set up the table
echo "\n";
@@ -181,4 +186,5 @@
write_header_cell ("list", "stdev ");
write_header_cell ("list", "Date/Time");
+write_header_cell ("list", "Cam FWHM_major");
echo "\n";
@@ -193,4 +199,5 @@
write_query_row ('rawExp.bg_stdev', 5, 'min');
write_query_row ('rawExp.dateobs', 19, 'min');
+write_query_row ('camProcessedExp.fwhm_major', 5, 'min');
echo "| ≤ | \n";
echo " | \n";
@@ -201,4 +208,5 @@
write_query_row ('rawExp.bg_stdev', 5, 'max');
write_query_row ('rawExp.dateobs', 19, 'max');
+write_query_row ('camProcessedExp.fwhm_major', 5, 'max');
echo "
\n";
@@ -228,5 +236,5 @@
// query the database
-$sql = "SELECT rawExp.ra,rawExp.decl,rawExp.filter FROM camRun, chipRun, rawExp $WHERE";
+$sql = "SELECT rawExp.ra,rawExp.decl,rawExp.filter FROM camProcessedExp, camRun, chipRun, rawExp $WHERE";
$qry = $db->query($sql);
Index: /branches/eam_branches/ipp-20120905/ippMonitor/raw/skycellplot.php
===================================================================
--- /branches/eam_branches/ipp-20120905/ippMonitor/raw/skycellplot.php (revision 34466)
+++ /branches/eam_branches/ipp-20120905/ippMonitor/raw/skycellplot.php (revision 34466)
@@ -0,0 +1,70 @@
+";
+
+ $newpath = getenv("PATH");
+ echo "new path: $newpath
";
+
+ $newpath = getenv("LD_LIBRARY_PATH");
+ echo "new path: $newpath
";
+
+ echo "bindir: $BINDIR
";
+}
+
+if ($debug) {
+ exec ("which skycellplot.dvo", $output, $status);
+ echo "which output:
";
+ for ($i = 0; $i < count($output); $i++) {
+ echo "output $i: $output[$i]
";
+ }
+ echo "status: $status
";
+}
+
+$infile = $_GET[input];
+$outfile = $_GET[output];
+
+$output = shell_exec("skycellplot.dvo $infile $outfile");//, $output, $status);
+
+if ($debug) {
+ for ($i = 0; $i < count($output); $i++) {
+ echo "output $i: $output[$i]
";
+ }
+ echo "status: $status
";
+ exit ();
+}
+
+# use these to check the environment
+# passthru ("env");
+
+$file = fopen ($outfile, "r");
+if ($file && !$debug) {
+ header ('Content-Type: image/png');
+ fpassthru ($file);
+}
+
+exit();
+
+?>
Index: /branches/eam_branches/ipp-20120905/ippMonitor/scripts/skycellplot.dvo
===================================================================
--- /branches/eam_branches/ipp-20120905/ippMonitor/scripts/skycellplot.dvo (revision 34466)
+++ /branches/eam_branches/ipp-20120905/ippMonitor/scripts/skycellplot.dvo (revision 34466)
@@ -0,0 +1,87 @@
+#!/usr/bin/env dvo
+
+# this should prbably be just skyplot.dvo with added switch for deg or rad coords
+#
+
+list filters
+ green
+ blue
+ red
+ indigo
+ black
+ gold
+ orange
+end
+
+list pttype
+ 7
+ 0
+ 1
+ 2
+ 7
+ 3
+ 0
+end
+
+list ptsize
+ 1.1
+ 0.3
+ 0.3
+ 0.3
+ 0.3
+ 0.3
+ 0.3
+end
+
+macro skycellplot
+ if ($0 != 3)
+ echo "skycellplot (input) (output)"
+ break
+ end
+
+ data $1
+ read r 1 d 2 f 3
+
+ resize 1000 500
+ region +ns -ew 180 0 85 ait; cgrid -ra-by-hour -c grey80
+
+ set R = r
+ set D = d
+
+ for i 0 $filters:n
+ subset R1 = R if (f == $i)
+ subset D1 = D if (f == $i)
+ if (R1[])
+ cplot R1 D1 -pt $pttype:$i -sz $ptsize:$i -c $filters:$i
+ end
+ end
+
+ style -c red -pt 0 -sz 0.5; ecliptic
+ style -c blue -pt 0 -sz 0.5; galactic
+
+ png -name $2
+end
+
+macro galactic
+ create l 720 -360 0.1
+ set b = l * 0
+ csystem G C l b
+ cplot l b
+end
+
+macro ecliptic
+ create l 720 -360 0.1
+ set b = l * 0
+ csystem E C l b
+ cplot l b
+end
+
+if ($SCRIPT)
+ $KAPA = kapa -noX
+ if ($argv:n != 2)
+ echo "USAGE: skycellplot (input) (output)"
+ exit 1
+ end
+ skycellplot $argv:0 $argv:1
+ exit 0
+end
Index: /branches/eam_branches/ipp-20120905/ippToPsps/jython/objectbatch.py
===================================================================
--- /branches/eam_branches/ipp-20120905/ippToPsps/jython/objectbatch.py (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ippToPsps/jython/objectbatch.py (revision 34466)
@@ -124,9 +124,10 @@
," + filter[1] + "StackPSFFlux = FLUX_PSF \
," + filter[1] + "StackPSFFluxErr = FLUX_PSF_ERR \
- ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) -8.9 \
+ ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) + 8.9 \
," + filter[1] + "StackKronMagErr = FLUX_KRON_ERR / FLUX_KRON \
," + filter[1] + "20pct = MAG_20/1000 \
," + filter[1] + "80pct = MAG_80/1000 \
- "
+ ," + filter[1] + "Flags = FLAGS "
+
self.scratchDb.execute(sql)
@@ -141,4 +142,20 @@
WHERE " + filter[1] + "MeanPSFMagErr > " + str(cut)
self.scratchDb.execute(sql)
+
+ self.logger.infoPair("Setting to NULL all x20pct =", "32.767" )
+ for filter in filters:
+ sql = "UPDATE Object \
+ SET " + filter[1] + "20pct = null \
+ WHERE " + filter[1] + "20pct > 32.766"
+ self.scratchDb.execute(sql)
+ self.logger.infoPair("Setting to NULL all x80pct =", "32.767" )
+ for filter in filters:
+ sql = "UPDATE Object \
+ SET " + filter[1] + "80pct = null \
+ WHERE " + filter[1] + "80pct > 32.766"
+ self.scratchDb.execute(sql)
+
+
+
'''
@@ -187,4 +204,6 @@
,decErr \
,nDetections \
+ ,extra \
+ ,sgSep \
,dataRelease \
) \
@@ -200,4 +219,6 @@
,DEC_ERR \
,NMEASURE \
+ ,PSF_QF_PERF \
+ ,STARGAL_SEP \
, " + str(self.config.dataRelease) + "\
FROM " + cptTableName
Index: /branches/eam_branches/ipp-20120905/ippToPsps/jython/scratchdb.py
===================================================================
--- /branches/eam_branches/ipp-20120905/ippToPsps/jython/scratchdb.py (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ippToPsps/jython/scratchdb.py (revision 34466)
@@ -391,5 +391,5 @@
except:
self.logger.error("Unable to create DVO detection database table")
-
+ self.makeColumnUnique("dvoDetection", "objID")
'''
Index: /branches/eam_branches/ipp-20120905/ippToPsps/jython/stackbatch.py
===================================================================
--- /branches/eam_branches/ipp-20120905/ippToPsps/jython/stackbatch.py (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ippToPsps/jython/stackbatch.py (revision 34466)
@@ -374,5 +374,5 @@
,PSF_THETA \
,psfLikelihood(EXT_NSIGMA) \
- ,FLAGS << 32 | FLAGS2 \
+ ,FLAGS2 << 32 | FLAGS \
,PSF_QF \
,MOMENTS_XX \
@@ -389,5 +389,5 @@
,KRON_FLUX / " + str(self.expTime) + " \
,KRON_FLUX_ERR / " + str(self.expTime) + " \
- , NULL \
+ , MOMENTS_R1 * 2.5 \
, NULL \
,N_FRAMES \
@@ -395,6 +395,6 @@
," + self.historyModNum + " \
FROM SkyChip_psf"
-
- self.scratchDb.execute(sql)
+ self.scratchDb.execute(sql)
+
#it is possible to drop some detections from dvo (that are present in the cmf). when that happens we get a 0 for objid
#we drop those...
@@ -402,14 +402,23 @@
self.scratchDb.execute(sql)
self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0")
-
+
self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID))
+
self.scratchDb.updateFilterID("StackDetection", self.filter)
+
self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.config.dataRelease))
+
self.scratchDb.updateAllRows("StackDetection", "primaryF", "0")
+
self.scratchDb.updateAllRows("StackDetection", "activeFlag", "0")
+
self.updateStackMetaID("StackDetection")
+
self.updateStackTypeID("StackDetection")
+
self.updateDvoIDs("StackDetection")
+
sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)"
+
self.scratchDb.execute(sql)
@@ -422,12 +431,17 @@
AND a.ippDetectID = b.IPP_IDET \
AND b.PSF_INST_FLUX_SIG IS NOT NULL"
+
self.scratchDb.execute(sql)
+
self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux")
+
self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID")
+
sql="DELETE FROM StackDetection where objID = 0"
- self.scratchDb.execute(sql)
- self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0")
-
+
+ self.scratchDb.execute(sql)
+ self.logger.infoPair("Delleting", "entries with StackDetection.objID = 0")
+
'''
@@ -600,4 +614,5 @@
self.logger.debug("Altering PSPS tables")
+ # heather uncommented the unique line -- well that no worky
#self.scratchDb.makeColumnUnique("StackDetection", "objID")
self.scratchDb.createIndex("StackDetection", "ippDetectID")
@@ -624,6 +639,7 @@
'''
def updateDvoIDs(self, table):
-
+ self.logger.infoPair("getting","imageID")
imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID'])
+ self.logger.infoPair("obtained","imageID")
self.logger.debug("Updating table '" + table + "' with DVO IDs...")
sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \
@@ -633,6 +649,7 @@
WHERE a.ippDetectID = b.ippDetectID \
AND b.imageID = " + str(imageID)
- self.scratchDb.execute(sql)
-
+
+ self.scratchDb.execute(sql)
+
'''
Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
Index: /branches/eam_branches/ipp-20120905/ippTools/share/difftool_definestackstack_part0.sql
===================================================================
--- /branches/eam_branches/ipp-20120905/ippTools/share/difftool_definestackstack_part0.sql (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ippTools/share/difftool_definestackstack_part0.sql (revision 34466)
@@ -39,5 +39,5 @@
WHERE 1
@STACK1_QUERY@ -- input condition
-) AS diffExp ON diffExp.stack1 = stackRun.stack_id AND diffExp.stack2 = template.max_stack_id
+) AS diffExp ON diffExp.stack1 = stackRun.stack_id
WHERE
stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0
Index: /branches/eam_branches/ipp-20120905/ippTools/share/difftool_definestackstack_part1.sql
===================================================================
--- /branches/eam_branches/ipp-20120905/ippTools/share/difftool_definestackstack_part1.sql (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ippTools/share/difftool_definestackstack_part1.sql (revision 34466)
@@ -40,5 +40,5 @@
WHERE 1
@STACK1_QUERY@ -- input constraint
-) AS diffExp ON diffExp.stack1 = stackRun.stack_id AND diffExp.stack2 = template.max_stack_id
+) AS diffExp ON diffExp.stack1 = stackRun.stack_id
WHERE
stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0
Index: /branches/eam_branches/ipp-20120905/ippTools/src/difftoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20120905/ippTools/src/difftoolConfig.c (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ippTools/src/difftoolConfig.c (revision 34466)
@@ -351,5 +351,5 @@
psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if no stacks available for some skycells", false);
- psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false);
+ psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template and same template label", false);
psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false);
Index: /branches/eam_branches/ipp-20120905/ppTranslate/src/ppMopsWrite.c
===================================================================
--- /branches/eam_branches/ipp-20120905/ppTranslate/src/ppMopsWrite.c (revision 34465)
+++ /branches/eam_branches/ipp-20120905/ppTranslate/src/ppMopsWrite.c (revision 34466)
@@ -38,9 +38,30 @@
// Get these header words from the first input non null input
+ // but for SEEING which is the median of the (non-NULL) detections
+ // seeings
ppMopsDetections *det = NULL;
- for (int d = 0; d < detections->n && det == NULL; d++) {
- det = detections->data[d];
+ int validSeeingCount = 0;
+ for (int d = 0; d < detections->n; d++) {
+ if (detections->data[d] != NULL) {
+ if (det == NULL) {
+ det = detections->data[d];
+ }
+ if (isfinite(((ppMopsDetections*) detections->data[d])->seeing)) {
+ validSeeingCount += 1;
+ }
+ }
}
if (det != NULL) {
+ //Get the SEEING median
+ psVector* seeingValues = psVectorAlloc(validSeeingCount, PS_TYPE_F32);
+ psVectorInit(seeingValues, NAN);
+ validSeeingCount = 0;
+ for (int d = 0; d < detections->n; d++) {
+ if ( (detections->data[d] != NULL) && (isfinite(((ppMopsDetections*) detections->data[d])->seeing)) ){
+ seeingValues->data.F32[validSeeingCount++] = ((ppMopsDetections*) detections->data[d])->seeing;
+ }
+ }
+ psStats *seeingStats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN);
+ psVectorStats(seeingStats, seeingValues, NULL, NULL, 0);
psMetadataAddF64(header, PS_LIST_TAIL, "MJD-OBS", 0, "MJD of exposure midpoint", det->mjd);
psMetadataAddStr(header, PS_LIST_TAIL, "RA", 0, "Right Ascension of boresight", det->raBoresight);
@@ -52,5 +73,5 @@
psMetadataAddStr(header, PS_LIST_TAIL, "FILTER", 0, "Filter name", det->filter);
psMetadataAddF32(header, PS_LIST_TAIL, "AIRMASS", 0, "Airmass of exposure", det->airmass);
- psMetadataAddF32(header, PS_LIST_TAIL, "SEEING", 0, "Mean seeing", det->seeing);
+ psMetadataAddF32(header, PS_LIST_TAIL, "SEEING", 0, "Mean seeing", seeingStats->sampleMedian);
} else {
psWarning("no inputs with surviving detections. output header will be incomplete");
@@ -112,59 +133,59 @@
// Write data of version 2 (see ICD)
psMetadataAdd (row, PS_LIST_TAIL, "IPP_IDET", PS_DATA_U32, "IPP detection identifier index",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX", PS_DATA_F32, "PSF fit instrumental magnitude",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX_SIG",PS_DATA_F32, "Sigma of PSF instrumental magnitude",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG", PS_DATA_F32, "magnitude in standard aperture",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RAW", PS_DATA_F32, "magnitude in real aperture",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS", PS_DATA_F32, "radius used for aperture mags",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "AP_FLUX", PS_DATA_F32, "instrumental flux in standard aperture",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "AP_FLUX_SIG", PS_DATA_F32, "aperture flux error",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG", PS_DATA_F32, "PSF Magnitude using supplied calibration",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG_SIG", PS_DATA_F32, "measured scatter of zero point calibration",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "SKY", PS_DATA_F32, "Sky level",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "SKY_SIGMA", PS_DATA_F32, "Sigma of sky level",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "PSF_QF_PERFECT", PS_DATA_F32, "PSF coverage/quality factor (poor)",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_R1", PS_DATA_F32, "first radial moment",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_RH", PS_DATA_F32, "half radial moment",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX", PS_DATA_F32, "Kron Flux (in 2.5 R1)",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_ERR", PS_DATA_F32, "Kron Flux Error",
- NAN);
- psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_INNER", PS_DATA_F32, "Kron Flux (in 1.0 R1)",
- NAN);
+ NAN);
+ psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_INNER", PS_DATA_F32, "Kron Flux (in 1.0 R1)",
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_OUTER", PS_DATA_F32, "Kron Flux (in 4.0 R1)",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "DIFF_R_P", PS_DATA_F32, "distance to positive match source",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "DIFF_SN_P", PS_DATA_F32, "signal-to-noise of pos match src",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "DIFF_R_M", PS_DATA_F32, "distance to negative match source",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "DIFF_SN_M", PS_DATA_F32, "signal-to-noise of neg match src",
- NAN);
+ NAN);
psMetadataAdd (row, PS_LIST_TAIL, "FLAGS2", PS_DATA_U32, "psphot analysis flags (group 2)",
- 0);
- psMetadataAdd (row, PS_LIST_TAIL, "N_FRAMES", PS_DATA_U16, "Number of frames overlapping source center",
- 0);
- psMetadataAdd (row, PS_LIST_TAIL, "PADDING", PS_DATA_S16, "padding",
- 0);
+ 0);
+ psMetadataAdd (row, PS_LIST_TAIL, "N_FRAMES", PS_DATA_U16, "Number of frames overlapping source center",
+ 0);
+ psMetadataAdd (row, PS_LIST_TAIL, "PADDING", PS_DATA_S16, "padding",
+ 0);
if (args->version == 3) {
- // TODO: Write data of version 3 (see ICD)
+ // TODO: Write data of version 3 (see ICD)
}
}
@@ -179,8 +200,8 @@
} else {
-#define addColumn(_outName, _inName, _convertTo32) \
+#define addColumn(_outName, _inName, _convertTo32) \
if (!addOutputColumn(table, detections, total, _outName, _inName, _convertTo32)) { \
psError(PS_ERR_UNKNOWN, false, "Failed to add column %s", _outName); \
- return false; \
+ return false; \
}
@@ -198,8 +219,8 @@
addColumn("EXT_SIGNIFICANCE", "EXT_NSIGMA", 0);
addColumn("PSF_MAJOR", NULL, 0);
- addColumn("PSF_MINOR", NULL, 0);
- addColumn("PSF_THETA", NULL, 0);
- addColumn("PSF_QUALITY", "PSF_QF", 0);
- addColumn("PSF_NPIX", NULL, 1);
+ addColumn("PSF_MINOR", NULL, 0);
+ addColumn("PSF_THETA", NULL, 0);
+ addColumn("PSF_QUALITY", "PSF_QF", 0);
+ addColumn("PSF_NPIX", NULL, 1);
addColumn("MOMENTS_XX", NULL, 0);
addColumn("MOMENTS_XY", NULL, 0);
@@ -287,32 +308,32 @@
ppMopsDetections *det = detections->data[i];
if (!det || det->num == 0) {
- // no detections survived for this input
- continue;
+ // no detections survived for this input
+ continue;
}
psVector *in = NULL;
in = psMetadataLookupVector(NULL, det->table, inColumnName);
if (!in) {
- psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s (convertTo32 is true)", inColumnName);
- return false;
+ psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s (convertTo32 is true)", inColumnName);
+ return false;
}
if (in->type.type != PS_TYPE_S64 && in->type.type != PS_TYPE_U64) {
- psError(PS_ERR_PROGRAMMING, true, "input column to convert is not S64 or U64: %s %d",
- inColumnName, in->type.type);
- return false;
+ psError(PS_ERR_PROGRAMMING, true, "input column to convert is not S64 or U64: %s %d",
+ inColumnName, in->type.type);
+ return false;
}
if (out == NULL) {
- // First time through set up the output vector and the copy parameters
- if (in->type.type == PS_TYPE_S64) {
- out = psVectorAlloc(outputSize, PS_TYPE_S32);
- } else {
- out = psVectorAlloc(outputSize, PS_TYPE_U32);
- }
+ // First time through set up the output vector and the copy parameters
+ if (in->type.type == PS_TYPE_S64) {
+ out = psVectorAlloc(outputSize, PS_TYPE_S32);
+ } else {
+ out = psVectorAlloc(outputSize, PS_TYPE_U32);
+ }
}
for (long d=0; d < det->num; d++) {
- if (in->type.type == PS_TYPE_S64) {
- out->data.S32[next++] = in->data.S64[d];
- } else {
- out->data.U32[next++] = in->data.U64[d];
- }
+ if (in->type.type == PS_TYPE_S64) {
+ out->data.S32[next++] = in->data.S64[d];
+ } else {
+ out->data.U32[next++] = in->data.U64[d];
+ }
}
}
@@ -323,57 +344,57 @@
ppMopsDetections *det = detections->data[i];
if (!det || det->num == 0) {
- // no detections survived for this input
- continue;
+ // no detections survived for this input
+ continue;
}
psVector *in = NULL;
in = psMetadataLookupVector(NULL, det->table, inColumnName);
if (!in) {
- psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s (convertTo32 is false)", inColumnName);
- out = psVectorAlloc(outputSize, PS_TYPE_F32);
- psVectorInit(out, NAN);
- psMetadataAddVector(table, PS_LIST_TAIL, outColumnName, 0, NULL, out);
- return false;
+ psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s (convertTo32 is false)", inColumnName);
+ out = psVectorAlloc(outputSize, PS_TYPE_F32);
+ psVectorInit(out, NAN);
+ psMetadataAddVector(table, PS_LIST_TAIL, outColumnName, 0, NULL, out);
+ return false;
}
if (out == NULL) {
- // First time through set up the output vector and the copy parameters
- out = psVectorAlloc(outputSize, in->type.type);
- next = (void *) out->data.U8;
- switch (in->type.type) {
- case PS_TYPE_S8:
- elementSize = sizeof(psS8);
- break;
- case PS_TYPE_U8:
- elementSize = sizeof(psU8);
- break;
- case PS_TYPE_S16:
- elementSize = sizeof(psS16);
- break;
- case PS_TYPE_U16:
- elementSize = sizeof(psU16);
- break;
- case PS_TYPE_S32:
- elementSize = sizeof(psS32);
- break;
- case PS_TYPE_U32:
- elementSize = sizeof(psU32);
- break;
- case PS_TYPE_S64:
- elementSize = sizeof(psS64);
- break;
- case PS_TYPE_U64:
- elementSize = sizeof(psU64);
- break;
- case PS_TYPE_F32:
- elementSize = sizeof(psF32);
- break;
- case PS_TYPE_F64:
- elementSize = sizeof(psF64);
- break;
- default:
- psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Unknown vector type %d", in->type.type);
- return false;
- }
- }
- // We are doing nasty things here so we can use memcpy.
+ // First time through set up the output vector and the copy parameters
+ out = psVectorAlloc(outputSize, in->type.type);
+ next = (void *) out->data.U8;
+ switch (in->type.type) {
+ case PS_TYPE_S8:
+ elementSize = sizeof(psS8);
+ break;
+ case PS_TYPE_U8:
+ elementSize = sizeof(psU8);
+ break;
+ case PS_TYPE_S16:
+ elementSize = sizeof(psS16);
+ break;
+ case PS_TYPE_U16:
+ elementSize = sizeof(psU16);
+ break;
+ case PS_TYPE_S32:
+ elementSize = sizeof(psS32);
+ break;
+ case PS_TYPE_U32:
+ elementSize = sizeof(psU32);
+ break;
+ case PS_TYPE_S64:
+ elementSize = sizeof(psS64);
+ break;
+ case PS_TYPE_U64:
+ elementSize = sizeof(psU64);
+ break;
+ case PS_TYPE_F32:
+ elementSize = sizeof(psF32);
+ break;
+ case PS_TYPE_F64:
+ elementSize = sizeof(psF64);
+ break;
+ default:
+ psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Unknown vector type %d", in->type.type);
+ return false;
+ }
+ }
+ // We are doing nasty things here so we can use memcpy.
// It would be safer to do a proper loop over the elements.
long toCopy = det->num * elementSize;
Index: /branches/eam_branches/ipp-20120905/pstamp/scripts/dquery_finish.pl
===================================================================
--- /branches/eam_branches/ipp-20120905/pstamp/scripts/dquery_finish.pl (revision 34465)
+++ /branches/eam_branches/ipp-20120905/pstamp/scripts/dquery_finish.pl (revision 34466)
@@ -128,4 +128,9 @@
my $job_id = $job->{job_id};
+ if ($job->{fault}) {
+ # how do we get this information back to the user?
+ print STDERR "job: $job_id faulted with $job->{fault}\n";
+ next;
+ }
my $outputBase = $job->{outputBase};
my $results = "${outputBase}results.txt";
@@ -166,5 +171,5 @@
my $results = shift;
- open IN, "<$results_file" or my_die("failed to open $results_file", $PS_EXIT_UNKNOWN_ERROR);
+ open IN, "<$results_file" or my_die("failed to open $results_file\n", $PS_EXIT_UNKNOWN_ERROR);
foreach my $line () {
Index: /branches/eam_branches/ipp-20120905/pstamp/scripts/psmkreq
===================================================================
--- /branches/eam_branches/ipp-20120905/pstamp/scripts/psmkreq (revision 34465)
+++ /branches/eam_branches/ipp-20120905/pstamp/scripts/psmkreq (revision 34466)
@@ -52,4 +52,11 @@
my $comment;
+my $survey_name = 'null';
+my $release = 'null';
+my $stack_type = 'null';
+my $bgrestore;
+my $jpeg;
+my $nocompress;
+
my $missing_tools;
my $pstamp_request_file = can_run('pstamp_request_file') or (warn "Can't find required program pstamp_request_file" and $missing_tools = 1);
@@ -68,5 +75,6 @@
'width=i' => \$width,
'height=i' => \$height,
- 'pixcenter' => \$pixcenter,
+# pixcenter is deprecated
+# 'pixcenter' => \$pixcenter,
'arcseconds' => \$arcseconds,
'coord_mask=i' => \$coord_mask,
@@ -87,9 +95,14 @@
'comment=s' => \$comment,
+ # new columns for version 2
+ 'survey=s' => \$survey_name,
+ 'release=s' => \$release,
+ 'stack_type' => \$stack_type,
+
'option_mask=i' => \$option_mask,
'image' => \$image,
'mask' => \$mask,
'variance' => \$variance,
- 'cmf' => \$cmf,
+ 'sources' => \$cmf,
'psf' => \$psf,
'backmdl' => \$backmdl,
@@ -99,4 +112,8 @@
'do_not_wait' => \$no_wait,
+ # new options for version 2
+ 'no_compress' => \$nocompress,
+ 'bgrestore' => \$bgrestore,
+
'verbose' => \$verbose,
'save-temps' => \$save_temps,
@@ -110,7 +127,7 @@
pod2usage( -msg => "Invalid job_type: $job_type", -exitval =>1 )
- unless ($job_type eq 'stamp' or $job_type eq 'get_image');
-
-if ($job_type eq 'stamp') {
+ unless ($job_type eq 'stamp' or $job_type eq 'get_image' or $job_type eq 'listtargets');
+
+if (($job_type eq 'stamp') or ($job_type eq 'listtargets')) {
if (defined $list) {
pod2usage( -msg => "--ra --dec --x --y are not used with --list", -exitval =>1 )
@@ -152,6 +169,7 @@
$id = 0 if !$id;
-unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') {
- die "$stage is not a valid value for stage\n";
+
+unless (($job_type eq 'listtargets' and $stage eq 'exposure') or $stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') {
+ die "$stage is not a valid stage for $job_type jobs\n";
}
@@ -207,16 +225,16 @@
} else {
$rows = [];
- push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max);
+ push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max, $survey_name, $release, $stack_type);
}
my ($tdf, $table_def_name) = tempfile ("/tmp/tabledef.XXXX", UNLINK => !$save_temps);
-print $tdf "$req_name 1\n";
+print $tdf "$req_name 2\n";
my $rownum = 0;
foreach my $row (@$rows) {
$rownum++;
my $line = "$rownum $row->{ra}\t$row->{dec}\t$width $height"
- . " $coord_mask $job_type $option_mask $project $req_type"
+ . " $coord_mask $job_type $option_mask $project $row->{survey_name} $row->{release} $req_type"
. " $stage $id $tess_id $component $data_group"
- . " $row->{filter} $row->{mjd_min} $row->{mjd_max}";
+ . " $row->{filter} $row->{mjd_min} $row->{mjd_max} $row->{stack_type}";
if ($row->{comment} and $row->{comment} ne '') {
@@ -230,4 +248,7 @@
}
close $tdf;
+
+system "cat $table_def_name";
+#exit 0;
{
@@ -277,6 +298,9 @@
$row->{dec} = checkDEC($vals[1], $linenumber);
$row->{filter} = checkFilter($vals[2], $filter, $linenumber);
- $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);;
- $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);;
+ $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);
+ $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);
+ $row->{survey_name} = $vals[5];
+ $row->{release} = $vals[6];
+ $row->{stack_type} = $vals[7];
$row->{comment} = $comment;
Index: /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_checkdependent.pl
===================================================================
--- /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_checkdependent.pl (revision 34465)
+++ /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_checkdependent.pl (revision 34466)
@@ -137,5 +137,5 @@
print "skipping $command\n";
}
-} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update')) {
+} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'goto_cleaned') or ($it->{state} eq 'update')) {
# For warp and diff stages we need to call the 'queue_update' subroutines even if the
# data_state is update in order to check the state of inputs in earlier stages in the pipeline
@@ -260,4 +260,25 @@
# caller will fault the jobs
return $error_code;
+ } elsif ($chip->{state} eq 'goto_cleaned') {
+
+ if ($chip->{data_state} eq 'full') {
+ # cleanup has been queued, but has not yet finished. (cleanup pantasks is probably stopped)
+ # Since this chip is in full go ahead and set the run to state update and change the label.
+ # XXX: In the very rare case that this chip run is actually in the running cleanup pantasks
+ # queue things may get confused but we can live with that
+ my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
+ $command .= " -set_label $rlabel" if $rlabel;
+
+ if (!$no_update) {
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ my_die("failed to change ${stage}Run $stage_id $component from goto_cleaned to update", $PS_EXIT_UNKNOWN_ERROR);
+ }
+ } else {
+ print "skipping $command\n";
+ }
+ $queued_update = 1;
+ }
} elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
@@ -344,4 +365,22 @@
my $skycell = $metadata;
+ if ($skycell->{state} eq 'goto_cleaned' and $skycell->{data_state} eq 'full') {
+ # cleanup has been queued, but hasn't been done. it's "safe" to set the state back to full
+ # XXX: seem comments in check_states_chip
+ my $command = "$warptool -updaterun -set_state update -warp_id $warp_id";
+ $command .= " -set_label $rlabel" if $rlabel;
+
+ if (!$no_update) {
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ my_die("failed to change state of ${stage}Run $stage_id from goto_cleaned to update", $PS_EXIT_UNKNOWN_ERROR);
+ }
+ } else {
+ print "skipping $command\n";
+ }
+ return 0;
+ }
+
# get the list of input chips for this skycell
my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id";
@@ -428,4 +467,21 @@
my $skycell = $metadata;
my $skycell_id = $skycell->{skycell_id};
+
+ if ($metadata->{state} eq 'goto_cleaned' and $metadata->{data_state} eq 'full') {
+ print "changing state of diffRun $diff_id from goto_cleaned to update. skycell $skycell_id is full\n";
+ my $command = "$difftool -updaterun -set_state update -diff_id $diff_id";
+ $command .= " -set_label $rlabel" if $rlabel;
+
+ if (!$no_update) {
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+ }
+ } else {
+ print "skipping $command\n";
+ }
+ return 0;
+ }
if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {
Index: /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_finish.pl
===================================================================
--- /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_finish.pl (revision 34465)
+++ /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_finish.pl (revision 34466)
@@ -107,7 +107,8 @@
my ($header, $rows) = read_request_file($req_file);
- if (!$header or !$rows) {
- # Since a request got queued, the request file must have been readable at some
- # point
+ my $action = $header->{ACTION};
+
+ if (!$header or (($action eq 'PROCESS') and !$rows)) {
+ # Since a request got queued, the request file must have been readable at some point
my_die("failed to read request file $req_file", $req_id, $PS_EXIT_CONFIG_ERROR);
}
@@ -126,152 +127,158 @@
}
- # The results table definition file
- my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
-
- # data for the header
- print $tdf "$req_name|$req_id|\n";
-
- # get the list of jobs generated for this request
- my @jobs;
- {
- my $command = "$pstamptool -listjob -req_id $req_id";
- $command .= " -dbname $dbname" if $dbname;
- $command .= " -dbserver $dbserver" if $dbserver;
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
- run(command => $command, verbose => $verbose);
- unless ($success) {
- die("Unable to perform $command error code: $error_code");
- }
- my $output = join "", @$stdout_buf;
- if (!$output) {
- # This should not happen. A fake job should have been entered
- my_die("Request $req_id produced no jobs", $req_id, $PS_EXIT_PROG_ERROR);
- } else {
- my $jobs = parse_md_fast($mdcParser, $output);
-
- @jobs = @$jobs;
- }
- }
-
- my $exp_info;
- foreach my $job (@jobs) {
- my $job_id = $job->{job_id};
- my $job_type = $job->{jobType};
- my $rownum = $job->{rownum};
- my $fault = $job->{fault};
- my $exp_id = $job->{exp_id};
-
- my $error_string = get_error_string($fault);
-
- if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
- # this request had a duplicate request name yet the parser didn't give
- # it an "ERROR style name.
- my_die("duplicate request not given a request name", $req_id, $PS_EXIT_PROG_ERROR);
- }
- my ($row, $req_info, $project) = get_request_info($rows, $rownum);
-
- my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
- my $image_db = $proj_hash->{dbname};
- if (!$image_db and !$fault) {
- # if project isn't resolvable, the paser should have faulted this job
- my_die("failed to find imagedb for project $project", $req_id, $PS_EXIT_CONFIG_ERROR);
- }
-
- my $job_params = get_job_parameters($job);
- my $stage = "";
- if ($job_params) {
- $stage = $job_params->{stage};
- }
-
- if ($stage ne 'stack') {
- # get the metadata for the exposure (if any i.e. stack)
- # returns an appropriate string if !$exp_id
- $exp_info = get_exposure_info($job_params, $image_db, $exp_id);
- } else {
- my $filter = $job_params->{filter};
- $filter = "0" if !$filter;
- my $mjd_obs = $job_params->{mjd_obs};
- $mjd_obs = "0" if !$mjd_obs;
- $exp_info = "$mjd_obs|0|0|$filter|0|0";
- }
-
- if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) {
- my $jreglist = "$outdir/reglist$job_id";
- if (open JRL, "<$jreglist") {;
- # process the reglist file to get the list of files produced by this job
- foreach my $line () {
-
- # XXX: we are getting many cases where the size and/or md5sum calculated by
- # the job has changed by the time the request_finish has run
- # Don't
- # add line to the requests's reglist
- # ....
-# print $rlf $line;
-
-
- chomp $line;
- my ($img_name, $reported_size, $reported_sum, $filetype) = split '\|', $line;
- # ... instead let dsreg compute the paramters by leaving them blank
- print $rlf "$img_name|||$filetype|\n";
-
- # add line to the table definition file
- print $tdf "$rownum|$fault|$error_string|$img_name|$job_id|";
-
- # ra_deg and dec_deg are the coordinates of center of the stamp
- # first assume that the image is compressed and check the first extension.
- # If not found check the PHU. If that doesn't work just set them to zero.
- # XXX do this more cleanly
- my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
- if (!defined $ra_deg) {
- (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`;
+ my $request_fault = 0;
+ if ($action eq 'PROCESS') {
+ # The results table definition file
+ my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
+ # data for the header
+ print $tdf "$req_name|$req_id|\n";
+ # get the list of jobs generated for this request
+ my @jobs;
+ {
+ my $command = "$pstamptool -listjob -req_id $req_id";
+ $command .= " -dbname $dbname" if $dbname;
+ $command .= " -dbserver $dbserver" if $dbserver;
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ die("Unable to perform $command error code: $error_code");
+ }
+ my $output = join "", @$stdout_buf;
+ if (!$output) {
+ # This should not happen. A fake job should have been entered
+ my_die("Request $req_id produced no jobs", $req_id, $PS_EXIT_PROG_ERROR);
+ } else {
+ my $jobs = parse_md_fast($mdcParser, $output);
+
+ @jobs = @$jobs;
+ }
+ }
+
+ my $exp_info;
+ foreach my $job (@jobs) {
+ my $job_id = $job->{job_id};
+ my $job_type = $job->{jobType};
+ my $rownum = $job->{rownum};
+ my $fault = $job->{fault};
+ my $exp_id = $job->{exp_id};
+
+ my $error_string = get_error_string($fault);
+
+ if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
+ # this request had a duplicate request name yet the parser didn't give
+ # it an "ERROR style name.
+ my_die("duplicate request not given a request name", $req_id, $PS_EXIT_PROG_ERROR);
+ }
+ my ($row, $req_info, $project) = get_request_info($rows, $rownum);
+
+ my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+ my $image_db = $proj_hash->{dbname};
+ if (!$image_db and !$fault) {
+ # if project isn't resolvable, the paser should have faulted this job
+ my_die("failed to find imagedb for project $project", $req_id, $PS_EXIT_CONFIG_ERROR);
+ }
+
+ my $job_params = get_job_parameters($job);
+ my $stage = "";
+ if ($job_params) {
+ $stage = $job_params->{stage};
+ }
+
+ if ($stage ne 'stack') {
+ # get the metadata for the exposure (if any i.e. stack)
+ # returns an appropriate string if !$exp_id
+ $exp_info = get_exposure_info($job_params, $image_db, $exp_id);
+ } else {
+ my $filter = $job_params->{filter};
+ $filter = "0" if !$filter;
+ my $mjd_obs = $job_params->{mjd_obs};
+ $mjd_obs = "0" if !$mjd_obs;
+ $exp_info = "$mjd_obs|0|0|$filter|0|0";
+ }
+
+ if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) {
+ my $jreglist = "$outdir/reglist$job_id";
+ if (open JRL, "<$jreglist") {;
+ # process the reglist file to get the list of files produced by this job
+ foreach my $line () {
+
+ # XXX: we are getting many cases where the size and/or md5sum calculated by
+ # the job has changed by the time the request_finish has run
+ # Don't
+ # add line to the requests's reglist
+ # ....
+ # print $rlf $line;
+
+
+ chomp $line;
+ my ($img_name, $reported_size, $reported_sum, $filetype) = split '\|', $line;
+ # ... instead let dsreg compute the paramters by leaving them blank
+ print $rlf "$img_name|||$filetype|\n";
+
+ # add line to the table definition file
+ print $tdf "$rownum|$fault|$error_string|$img_name|$job_id|";
+
+ # ra_deg and dec_deg are the coordinates of center of the stamp
+ # first assume that the image is compressed and check the first extension.
+ # If not found check the PHU. If that doesn't work just set them to zero.
+ # XXX do this more cleanly
+ my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
+ if (!defined $ra_deg) {
+ (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`;
+ }
+ $ra_deg = 0.0 if (!$ra_deg);
+ $dec_deg = 0.0 if (!$dec_deg);
+ print $tdf "$ra_deg|$dec_deg|";
+
+ print $tdf "$exp_info|";
+ print $tdf "$req_info|";
+ print $tdf "\n";
}
- $ra_deg = 0.0 if (!$ra_deg);
- $dec_deg = 0.0 if (!$dec_deg);
- print $tdf "$ra_deg|$dec_deg|";
-
+ close JRL;
+ } else {
+ my_die("No reglist for successful job: $job_id", $req_id, $PS_EXIT_PROG_ERROR)
+ if $fault eq $PSTAMP_SUCCESS;
+ print STDERR "no reglist file for job $job_id\n" if $verbose;
+ print $tdf "$rownum|$fault|$error_string|0|$job_id|";
+ print $tdf "0|0|"; # center of (non-existent) stamp
print $tdf "$exp_info|";
print $tdf "$req_info|";
print $tdf "\n";
}
- close JRL;
} else {
- my_die("No reglist for successful job: $job_id", $req_id, $PS_EXIT_PROG_ERROR)
- if $fault eq $PSTAMP_SUCCESS;
- print STDERR "no reglist file for job $job_id\n" if $verbose;
- print $tdf "$rownum|$fault|$error_string|0|$job_id|";
- print $tdf "0|0|"; # center of (non-existent) stamp
- print $tdf "$exp_info|";
- print $tdf "$req_info|";
- print $tdf "\n";
- }
- } else {
- # XXX do list jobs
- # we can probably arange things to use the code as above and skip the fileset registration
- print STDERR "Unknown jobType: $job_type";
- next;
- }
- }
- close $rlf;
- close $tdf;
- my $request_fault = 0;
- # make the results file
- {
- my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits";
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
- run(command => $command, verbose => $verbose);
- unless ($success) {
- print STDERR "Unable to perform $command error code: $error_code\n";
- $request_fault = $error_code >> 8;
- } else {
- # dump a textual representation
- my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc";
+ # XXX do list jobs
+ # we can probably arange things to use the code as above and skip the fileset registration
+ print STDERR "Unknown jobType: $job_type";
+ next;
+ }
+ }
+ close $tdf;
+ # make the results file
+ {
+ my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits";
my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
run(command => $command, verbose => $verbose);
unless ($success) {
+ print STDERR "Unable to perform $command error code: $error_code\n";
$request_fault = $error_code >> 8;
- my_die("Unable to perform $command error code: $error_code", $req_id, $request_fault);
- }
- }
- }
+ } else {
+ # dump a textual representation
+ my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ $request_fault = $error_code >> 8;
+ my_die("Unable to perform $command error code: $error_code", $req_id, $request_fault);
+ }
+ }
+ }
+ } elsif ($action ne 'LIST') {
+ my_die("Unexpected action $action found", $req_id, $PS_EXIT_PROG_ERROR);
+ } else {
+ # pstampparse did all of the work
+ }
+
+ close $rlf;
+
if (!$request_fault) {
# register the fileset
Index: /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_request_file
===================================================================
--- /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_request_file (revision 34465)
+++ /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_request_file (revision 34466)
@@ -49,8 +49,57 @@
value => undef
},
+ { name => 'ACTION',
+ writetype => TSTRING,
+ comment => 'Action to perform: LIST or PROCESS',
+ value => undef
+ },
+ { name => 'USERNAME',
+ writetype => TSTRING,
+ comment => 'username for request (optional)',
+ value => undef
+ },
+ { name => 'EMAIL',
+ writetype => TSTRING,
+ comment => 'email for submittor (optional)',
+ value => undef
+ },
];
# Specification of columns to write
my $columns = [
+ { name => 'ROWNUM', type => 'J', writetype => TULONG },
+
+ { name => 'CENTER_X', type => 'D', writetype => TDOUBLE },
+ { name => 'CENTER_Y', type => 'D', writetype => TDOUBLE },
+ { name => 'WIDTH', type => 'D', writetype => TDOUBLE },
+ { name => 'HEIGHT', type => 'D', writetype => TDOUBLE },
+ # 2 bits in COORD_MASK indicate what units of roi coords are
+ { name => 'COORD_MASK', type => 'J', writetype => TULONG },
+
+ { name => 'JOB_TYPE', type => '16A', writetype => TSTRING },
+ { name => 'OPTION_MASK',type => 'J', writetype => TULONG },
+
+ # image selection parameters
+ { name => 'PROJECT', type => '16A', writetype => TSTRING },
+ # new in version 2
+ { name => 'SURVEY_NAME',type => '16A', writetype => TSTRING },
+ { name => 'IPP_RELEASE', type => '16A', writetype => TSTRING },
+ # end of new in version 2
+ { name => 'REQ_TYPE', type => '16A', writetype => TSTRING },
+ { name => 'IMG_TYPE', type => '16A', writetype => TSTRING },
+ { name => 'ID', type => '16A', writetype => TSTRING },
+ { name => 'TESS_ID', type => '64A', writetype => TSTRING },
+ { name => 'COMPONENT', type => '64A', writetype => TSTRING },
+
+ { name => 'DATA_GROUP ',type => '64A', writetype => TSTRING },
+
+ { name => 'REQFILT', type => '16A', writetype => TSTRING },
+ { name => 'MJD_MIN', type => 'D', writetype => TDOUBLE },
+ { name => 'MJD_MAX', type => 'D', writetype => TDOUBLE },
+ { name => 'RUN_TYPE', type => '16A', writetype => TSTRING },
+
+ { name => 'COMMENT ', type => '64A', writetype => TSTRING },
+];
+my $columns_v1 = [
{ name => 'ROWNUM', type => 'J', writetype => TULONG },
@@ -89,4 +138,6 @@
}
+# build array of arrays for the column data. Note if the input file is version 1 then
+# some of thes will get dropped
my @colData;
foreach (@$columns) {
@@ -96,5 +147,6 @@
my $minimum_cols = 6;
-my $numRows = read_data_for_table($in,'\s+', \@colData, $header, $minimum_cols);
+my $extver = 2;
+my $numRows = read_data_for_table($in,'\s+', \@colData, $header, $minimum_cols, \$extver);
if (!$numRows) {
print STDERR "no data in $input\n";
@@ -115,5 +167,5 @@
$output = $req_name . ".fits" if !$output;
-my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header);
+my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $extver > 1 ? $columns : $columns_v1, $header);
exit $status;
@@ -230,4 +282,5 @@
my $header = shift; # rerence to array of header keyword descriptions
my $minimum_required_vals = shift;
+ my $r_extver = shift; # reference to extension version
my $line_num = 0;
@@ -243,13 +296,34 @@
my @vals = split /$sep/, $line;
my $nvals = @vals;
- die "number of header columns in input $nvals does not equal expected number of header words $nhead"
+ die "two few columns $nvals in header definition. Require at least 2" if @vals < 2;
+
+ $$r_extver = $vals[1];
+ if ($extver > 1) {
+ die "number of header columns in input $nvals does not equal expected number of header words $nhead"
if (@vals != @$header);
-
- for (my $i=0; $i < @$header; $i++) {
- $header->[$i]->{value} = $vals[$i];
- }
-
- last; # only one header line
- }
+ } else {
+ # drop header keywords not used in version 1
+ for (my $i = @$header - 1; $i >= $nvals; $i--) {
+ pop @$header;
+ }
+ }
+
+ for (my $i=0; $i < $nvals; $i++) {
+ my $val = $vals[$i];
+ if (defined $val) {
+ $header->[$i]->{value} = $val;
+ } else {
+ $header->[$i]->{value} = 'null';
+ }
+ }
+
+ last; # one line defines the header
+ }
+ }
+ if ($$r_extver == 1) {
+ # pop off the colData arrays to account for the unused columns
+ pop @colData;
+ pop @colData;
+ pop @colData;
}
@@ -271,4 +345,6 @@
exit 1;
}
+ # trim leading whitespace from comment
+ $comment =~ s/^\s+//;
if ($comment) {
if ($comment =~ /#/) {
@@ -381,7 +457,7 @@
print "If --output is omitted the output file name is set to REQ_NAME.fits\n\n";
print "Header 1 Line. Format:\n\n";
- print " REQ_NAME EXTVER\n\n";
+ print "REQ_NAME EXTVER ACTION USERNAME EMAIL\n\n";
print "REQUEST specification (1 or more lines). Format:\n\n";
- print " ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX | COMMENT\n\n";
+ print "ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE | COMMENT\n\n";
exit 0;
Index: /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_targetlist_file.pl
===================================================================
--- /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_targetlist_file.pl (revision 34466)
+++ /branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_targetlist_file.pl (revision 34466)
@@ -0,0 +1,292 @@
+#!/bin/env perl
+
+# program to create a fits binary table with EXTNAME PS1_PS_TARGETLIST
+
+use warnings;
+use strict;
+
+use Astro::FITS::CFITSIO qw( :constants );
+Astro::FITS::CFITSIO::PerlyUnpacking(1);
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use Math::Trig;
+use Data::Dumper;
+
+use constant EXTNAME => 'PS1_PS_TARGETLIST'; # Extension name for output table
+use constant EXTVER => 2;
+
+my ( $input, # Name of input Detectabilty Query table
+ $output, # Name of output table
+ $save_temps, # Save temporary files?
+ );
+
+GetOptions(
+ 'input|i=s' => \$input,
+ 'output|o=s' => \$output,
+ 'save-temps' => \$save_temps,
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --input --output",
+ -exitval => 3)
+ unless defined $input
+ and defined $output;
+
+
+# The header kewords
+my $header = [
+ { name => 'REQ_NAME',
+ writetype => TSTRING,
+ comment => 'Postage Stamp Request Name',
+ value => undef
+ },
+ { name => 'REQ_ID',
+ writetype => TLONGLONG,
+ comment => 'Postage Stamp Server Request id',
+ value => undef
+ },
+];
+
+# Specification of columns to write
+my $columns = [
+ # rownum from original request
+ { name => 'ROWNUM', type => 'V', writetype => TULONG },
+
+ # single exposure results
+ { name => 'FPA_ID', type => '20A', writetype => TSTRING },
+ { name => 'EXP_ID', type => 'V', writetype => TULONG },
+ { name => 'CHIP_ID', type => 'V', writetype => TULONG },
+ { name => 'CHIP_READY', type => '2A', writetype => TSTRING},
+# { name => 'CAM_ID', type => 'V', writetype => TULONG },
+ { name => 'WARP_ID', type => 'V', writetype => TULONG },
+ { name => 'WARP_READY', type => '2A', writetype => TSTRING},
+
+ # stack columns
+ { name => 'STACK_ID', type => 'V', writetype => TULONG },
+ { name => 'STACK_FAILED', type => '2A', writetype => TSTRING},
+ { name => 'SEEING', type => 'D', writetype => TDOUBLE },
+
+ # stack and diff specific columns
+ { name => 'TESS_ID', type => '64A', writetype => TSTRING },
+ { name => 'SKYCELL_ID', type => '64A', writetype => TSTRING },
+ { name => 'RUN_TYPE', type => '64A', writetype => TSTRING },
+
+ # actual start time of exposure
+ { name => 'MJD_OBS', type => 'D', writetype => TDOUBLE },
+ # field center at midpoint of expsure, in degrees
+ { name => 'RA_OBS', type => 'D', writetype => TDOUBLE },
+ # field center at midpoint of expsure, in degrees
+ { name => 'DEC_OBS', type => 'D', writetype => TDOUBLE },
+ # actual filter
+ { name => 'FILTER', type => '16A', writetype => TSTRING },
+ # exposure time of parent image
+ { name => 'EXPTIME', type => 'D', writetype => TDOUBLE },
+
+ # the following are copied from the original pstamp request
+
+ # image selection parameters
+ { name => 'PROJECT', type => '16A', writetype => TSTRING },
+ { name => 'SURVEY_NAME',type => '64A', writetype => TSTRING },
+ { name => 'IPP_RELEASE',type => '64A', writetype => TSTRING },
+ { name => 'REQ_TYPE', type => '16A', writetype => TSTRING },
+ { name => 'IMG_TYPE', type => '16A', writetype => TSTRING },
+ { name => 'ID', type => '16A', writetype => TSTRING },
+ { name => 'DATA_GROUP', type => '64A', writetype => TSTRING },
+
+ # error code from processing this row
+ { name => 'ERROR_CODE',type => 'V', writetype => TULONG },
+ # error string correspoding to ERROR_CODE
+ { name => 'ERROR_STR',type => '24A', writetype => TSTRING },
+
+ # from original request
+ { name => 'COMMENT', type => '64A', writetype => TSTRING },
+
+];
+
+my $in;
+if ($input eq '-') {
+ $in = \*STDIN;
+} else {
+ open $in, "<$input" or die "cannot open $input for reading";
+}
+
+my @colData;
+my $i = 0;
+foreach my $col (@$columns) {
+ print "$i $col->{name}\n";
+ push @colData, [];
+ $i++;
+}
+
+
+my $numRows = read_data_for_table($in,'\|', \@colData, $header);
+if (!$numRows) {
+ print STDERR "no data in $input\n";
+ exit 1;
+}
+
+my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header);
+
+exit $status;
+
+# TODO: put this in a module
+
+# two utility functions that may be used to create a FITS binary
+# table from hashes describing the header keywords and columns
+
+# read_table_description reads the data for a table from a simple text file
+# make_fits_table writes out the table to a named file
+
+
+# A function to build a fits binary table from supplied data
+#
+sub make_fits_table {
+ my $output = shift; # name of output file
+ my $extname = shift; # extension name
+ my $numRows = shift; # number of rows in the table
+ my $colData = shift; # ref to array of arrays containing the data for each column
+ my $columns = shift; # ref to array of column descriptions (each a hash)
+ # with keys: name, type, and writetype
+ my $header = shift; # ref to array of header keyword descriptions - each a hash
+ # with keys: name, name, writetype, comment, and value
+ my $status = 0;
+
+ die "incorrect arguments" if !defined($columns);
+ # note $header can be nil
+
+ # build arrays for cfitsio
+ my @colNames; # Names of columns
+ my @colTypes; # Types of columns
+ my @colWriteType; # type to use to write
+
+ foreach my $colSpec ( @$columns) {
+ push @colNames, $colSpec->{name};
+ push @colTypes, $colSpec->{type};
+ push @colWriteType, $colSpec->{writetype};
+ }
+
+ if (-e $output) {
+ unlink "$output" or die "failed to remove existing $output";
+ }
+
+ my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status ); # Output file handle
+ check_fitsio( $status );
+
+ $outFits->create_img( 16, 0, undef, $status );
+ check_fitsio( $status );
+
+ # Create the table
+
+ $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames,
+ \@colNames, \@colTypes, undef, $extname, $status );
+ check_fitsio( $status );
+
+ # if header keyword descriptions were provided add them
+ if ($header) {
+ foreach my $headerword ( @$header ) {
+ my $value = $headerword->{value};
+ unless (defined $value) {
+ print "Can't find header keyword $headerword\n";
+ next;
+ }
+ # zap quotation marks
+ $value =~ s/\'//g;
+ my $name = $headerword->{name};
+ my $type = $headerword->{writetype};
+ my $comment = $headerword->{comment};
+ $outFits->write_key( $type, $name, $value, $comment, $status );
+ check_fitsio( $status );
+ }
+ }
+
+
+ for (my $i = 0; $i < scalar @colNames; $i++) {
+ my $writeType = $colWriteType[$i];
+ $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData->[$i], $status );
+ check_fitsio( $status );
+ }
+
+ $outFits->close_file( $status );
+
+ return 0;
+
+} # end of sub make_fits_table
+
+
+
+# read the table contents from a file
+#
+# input text file format:
+# lines that begin with '#' are comment lines and are skipped.
+# other lines are data. Each data line is split into fields with the
+# provided separator
+#
+# if $header is not null header the first non-commented line is read to
+# fill the value for each header keyword. The number of fields must match
+# the number of keywords.
+#
+# Following the optional header data, each data line contains data for each
+# row in the table. The number of fields must match the number of column
+# arrays provided.
+
+sub read_data_for_table {
+ my $in = shift; # input file handle
+ my $sep = shift; # string containing field separator
+ my $colData = shift; # reference to an array of arrays for the data
+ my $header = shift; # rerence to array of header keyword descriptions
+
+ my $line_num = 0;
+
+ # read data for header if any data is expected
+ if ($header) {
+ my $nhead = @$header;
+ while (my $line = <$in>) {
+ $line_num++;
+ next if ($line =~ /^#/); # skip comment lines
+ chomp $line;
+ my @vals = split /$sep/, $line;
+ my $nvals = @vals;
+ die "number of header columns in input $nvals does not equal expected number of header words $nhead"
+ if (@vals != @$header);
+
+ for (my $i=0; $i < @$header; $i++) {
+ $header->[$i]->{value} = $vals[$i];
+ }
+
+ last; # only one header line
+ }
+ }
+
+ my $row_num = 0;
+ my $ncols = @$colData;
+ while (my $line = <$in>) {
+ $line_num++;
+ next if ($line =~ /^#/); # skip comment lines
+ chomp $line;
+
+ my @vals = split /$sep/, $line;
+ my $nvals = @vals;
+ die "number of columns $nvals in input does not equal expected number of header "
+ . " words $ncols on line $line_num" if ($nvals != $ncols);
+
+ for (my $col = 0; $col < @$colData; $col++) {
+ $colData->[$col]->[$row_num] = $vals[$col];
+ }
+ $row_num++;
+ }
+
+ # we return the number of rows read
+ return $row_num;
+}
+
+# From Astro::FITS::CFITSIO demo
+sub check_fitsio
+{
+ my $status = shift; # Status of FITSIO calls
+
+ if ($status != 0) {
+ my $msg; # Message to output
+ Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
+ die "CFITSIO error: $msg\n";
+ }
+}
Index: /branches/eam_branches/ipp-20120905/pstamp/scripts/pstampparse.pl
===================================================================
--- /branches/eam_branches/ipp-20120905/pstamp/scripts/pstampparse.pl (revision 34465)
+++ /branches/eam_branches/ipp-20120905/pstamp/scripts/pstampparse.pl (revision 34466)
@@ -93,5 +93,5 @@
my $fields_output;
{
- my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME";
+ my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USER EMAIL";
my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
run(command => $command, verbose => $verbose);
@@ -102,5 +102,5 @@
$fields_output = join "", @$stdout_buf;
}
-my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
+my (undef, $extname, $extver, $req_name, $action, $username, $email) = split " ", $fields_output;
# make sure the file contains what we are expecting
@@ -109,5 +109,5 @@
my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR) if (!$req_name);
-my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1");
+my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
@@ -230,5 +230,5 @@
# the images of interest for this new row doesn't match the list.
# process the list ...
- $num_jobs += processRows(\@rowList);
+ $num_jobs += processRows($action, \@rowList);
# and reset the list to contain just the new row
@@ -238,8 +238,8 @@
# out of rows process the list
if (scalar @rowList > 0) {
- $num_jobs += processRows(\@rowList);
-}
-
-if (($mode eq "queue_job") and ($num_jobs eq 0)) {
+ $num_jobs += processRows($action, \@rowList);
+}
+
+if (($action eq 'LIST' or $mode eq "queue_job") and ($num_jobs eq 0)) {
print STDERR "no jobs created for $req_name\n" if $verbose;
insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
@@ -387,7 +387,20 @@
}
-sub processRows {
+sub list_targets {
my $rowList = shift;
my $num_jobs = 0;
+
+ $num_jobs = 1;
+ return $num_jobs;
+}
+
+sub processRows {
+ my $action = shift;
+ my $rowList = shift;
+ my $num_jobs = 0;
+
+ if ($action eq 'LIST') {
+ return list_targets($rowList);
+ }
# all rows in the list are compatible