Index: /tags/ipp-20130307/DataStoreServer/web/php/getsmf.php =================================================================== --- /tags/ipp-20130307/DataStoreServer/web/php/getsmf.php (revision 35489) +++ /tags/ipp-20130307/DataStoreServer/web/php/getsmf.php (revision 35489) @@ -0,0 +1,133 @@ +$command\n
"; + } + $command = escapeshellcmd($command); + $output = array(); + + exec($command, $output, $command_status); + + if ($command_status == 0) { + // we only expect one line of output + $len = count($output); + if ($len == 1) { + list($filename, $pathname) = explode(" ", $output[0]); + if ($filename && $pathname) { + if (!$debug) { + $gotFile = 1; + } else { + echo "$filename $pathname\n"; + echo "$command\n"; + } + } + } else { + echo "unexpected output from $command: $output[0] $output[1]\n"; + } + } else { + if ($debug) { + echo "command failed $command_status\n"; + } + } +} else { +} + +if ($gotFile) { + if (!$rvar_list) { + // All systems are go. Time to write the output. + // First set up the header + header('Content-type: application/fits'); + header("Content-Disposition: attachment; filename=\"$filename\""); + $filesize = filesize($pathname); + header("Content-Length: $filesize"); + header('Expires: now'); + + // copy the contents of the file to the stream + readfile($pathname); + } else { + echo "smf file name is $filename
\nfile is $pathname\n"; + } + +} else { + // XXX: Figure out how to stop wget from redirecting these error + // messages to the nasty filename + $error_string="Could not find smf"; + if ($rvar_cam_id) { + $error_string .= " for cam_id: $rvar_cam_id"; + } elseif ($rvar_exp_name) { + $error_string .= " for exposure: $rvar_exp_name"; + } else { + $error_string .= ". No cam_id or exp_name provided"; + } + + echo "$error_string.\n"; +} + +function getVar($var) { + if ($_SERVER['REQUEST_METHOD'] == 'POST') { + $rvar = $_POST[$var]; + } else { + $rvar = $_GET[$var]; + } + $rvar = stripslashes($rvar); + $rvar = htmlentities($rvar); + $rvar = strip_tags($rvar); + return $rvar; +} + + +if ($list) { + // print lots of information + // phpinfo(-1); + + // print the most useful variables + // phpinfo(32); +} + +?> Index: /tags/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /tags/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35488) +++ /tags/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35489) @@ -15,4 +15,5 @@ our @EXPORT_OK = qw( locate_images + locate_images_for_row resolve_project getCamRunByCamID @@ -46,8 +47,11 @@ my $last_project = ""; -my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool); +my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool, $releasetool); my ($pstamptool, $dvoImagesAtCoords, $whichimage, $ppConfigDump); my $default_data_groups = ""; + +# old locate_images() function. Still used by dqueryparse +# Postage stamp server now uses locate_images_for_row() sub locate_images { @@ -56,5 +60,5 @@ my $rowList = shift; # required my $req_type = shift; # required - my $img_type = shift; # required + my $stage = shift; # required my $id = shift; # required unless req_type eq bycoord or byskycell my $tess_id = shift; @@ -78,5 +82,5 @@ ($req_type ne "byskycell"); - if (!$default_data_groups) { + if (!$data_group and !$default_data_groups) { $default_data_groups = load_data_groups($verbose); } @@ -106,14 +110,14 @@ my $x = $row->{CENTER_X}; my $y = $row->{CENTER_Y}; - my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); + my $results = lookup_bycoord($ipprc, $row, $imagedb, $stage, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); return $results; } - if (($req_type eq "byid") and ($img_type eq "diff")) { + if (($req_type eq "byid") and ($stage eq "diff")) { # lookups of all of the information for diff images requires a two level lookup to # get the exposure information. Switching the req_type allows us to keep that code # in one place $req_type = "bydiff"; - my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $img_type, $verbose); + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $stage, $verbose); return $results; } @@ -122,9 +126,9 @@ # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or # the image from the diffRun - my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $img_type, $verbose); + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $stage, $verbose); if (!$results) { return undef; } - if ($img_type eq "diff") { + if ($stage eq "diff") { # lookup_diff has done all of the work return $results; @@ -132,15 +136,15 @@ my $image = $results->[0]; - if ($img_type eq "raw") { + if ($stage eq "raw") { $req_type = "byid"; $id = $image->{exp_id}; return undef if !$id; # fall through and lookup byid - } elsif ($img_type eq "chip") { + } elsif ($stage eq "chip") { $req_type = "byid"; $id = $image->{chip_id}; return undef if !$id; # fall through and lookup byid - } elsif ($img_type eq "warp") { + } elsif ($stage eq "warp") { $req_type = "byid"; $id = $image->{warp_id}; @@ -148,5 +152,5 @@ return undef if !$id; # fall through and lookup by warp_id - } elsif ($img_type eq "stack") { + } elsif ($stage eq "stack") { $req_type = "byid"; $id = $image->{stack_id}; @@ -155,9 +159,9 @@ } else { # This is checked this elsewhere? - print STDERR "Error: $img_type is an unknown image type\n"; + print STDERR "Error: $stage is an unknown image type\n"; return undef; } } elsif ($req_type eq "byskycell") { - if (($img_type eq "raw") or ($img_type eq "chip")) { + if (($stage eq "raw") or ($stage eq "chip")) { print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n"; return undef; @@ -169,6 +173,145 @@ } - my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component, - $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); + my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component, + $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, + 0, 0, # fwhm cuts are not applied here + undef, undef, undef, $verbose); + + return $results; +} + +sub locate_images_for_row { + my $ipprc = shift; + my $imagedb = shift; + my $camera = shift; + my $row = shift; + my $verbose = shift; + + my $req_type = $row->{REQ_TYPE}; + + # we die in response to bad data in request files + # The caller is responsible for updating the database + # pstampparse.pl error checks now so this shouldn't happen + &my_die("Unknown req_type: $req_type", $PS_EXIT_PROG_ERROR) + if ($req_type ne "byid") and + ($req_type ne "byexp") and + ($req_type ne "bycoord") and + ($req_type ne "bydiff") and + ($req_type ne "byskycell"); + + my $stage = $row->{IMG_TYPE}; + my $id = $row->{ID}; + my $component = $row->{COMPONENT}; + my $tess_id = $row->{TESS_ID}; + + my $filter = $row->{REQFILT}; + my $mjd_min = $row->{MJD_MIN}; + my $mjd_max = $row->{MJD_MAX}; + my $data_group = $row->{DATA_GROUP}; + if (isnull($data_group) and !$default_data_groups) { + $default_data_groups = load_data_groups($verbose); + } + + my $rownum = $row->{ROWNUM}; + my $job_type = $row->{JOB_TYPE}; + my $option_mask= $row->{OPTION_MASK}; + + my $dateobs_begin; + my $dateobs_end; + if (!iszero($mjd_min)) { + $dateobs_begin = mjd_to_dateobs($mjd_min); + } + if (!iszero($mjd_max)) { + $dateobs_end = mjd_to_dateobs($mjd_max); + } + if (isnull($tess_id)) { + $tess_id = undef; + } + if (isnull($data_group)) { + $data_group = undef; + } + if (isnull($filter)) { + $filter = undef; + } + + if ($req_type eq "bycoord") { + my $x = $row->{CENTER_X}; + my $y = $row->{CENTER_Y}; + my $results = lookup_bycoord($ipprc, $row, $imagedb, $stage, $tess_id, $component, 0, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); + return $results; + } + + # for compatability with the lower level functions that haven't been converted to take a single row + my $rowList = [$row]; + + if (($req_type eq "byid") and ($stage eq "diff")) { + # lookups of all of the information for diff images requires a two level lookup to + # get the exposure information. Switching the req_type allows us to keep that code + # in one place + $req_type = "bydiff"; + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $stage, $verbose); + return $results; + } + + my ($release_name, $survey, $default_tess_id) = get_release_info($row); + if (!$tess_id and $default_tess_id) { + $tess_id = $default_tess_id; + } + + if ($req_type eq "bydiff") { + # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or + # the image from the diffRun + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $stage, $verbose); + if (!$results) { + return undef; + } + if ($stage eq "diff") { + # lookup_diff has done all of the work + return $results; + } + + my $image = $results->[0]; + if ($stage eq "raw") { + $req_type = "byid"; + $id = $image->{exp_id}; + return undef if !$id; + # fall through and lookup byid + } elsif ($stage eq "chip") { + $req_type = "byid"; + $id = $image->{chip_id}; + return undef if !$id; + # fall through and lookup byid + } elsif ($stage eq "warp") { + $req_type = "byid"; + $id = $image->{warp_id}; + $component = $image->{skycell_id}; + return undef if !$id; + # fall through and lookup by warp_id + } elsif ($stage eq "stack") { + $req_type = "byid"; + $id = $image->{stack_id}; + return undef if !$id; + # fall though and lookup by stack_id + } else { + # This is checked this elsewhere? + print STDERR "Error: $stage is an unknown image type\n"; + return undef; + } + } elsif ($req_type eq "byskycell") { + if (($stage eq "raw") or ($stage eq "chip")) { + print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n"; + return undef; + } + if (!$tess_id or !$component) { + print STDERR "component and tess_id are required for REQ_TYPE byskycell\n"; + return undef; + } + } + + my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component, + 0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, + getValOrZero($row->{FWHM_MIN}), getValOrZero($row->{FWHM_MAX}), + undef, $release_name, $survey, + $verbose); return $results; @@ -195,4 +338,9 @@ my $mjd_min = shift; my $mjd_max = shift; + my $fwhm_min = shift; + my $fwhm_max = shift; + my $selectedAstrom = shift; + my $release_name = shift; + my $survey = shift; my $verbose = shift; @@ -219,6 +367,22 @@ my $magic_arg = $need_magic ? " -destreaked" : ""; + my $row = $rowList->[0]; + # all rows have the same center type - my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); + my $skycenter = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); + + my $use_releasetool = 0; + my $release_args; + if (($req_type eq 'byexp' and $stage ne 'raw') or ($req_type eq 'byskycell' and $stage ne 'diff')) { + if ($release_name or $survey) { + $use_releasetool = 1; + if ($release_name) { + $release_args = " -release_name $release_name"; + } else { + $release_args = " -priority_order"; + } + $release_args .= " -surveyName $survey" if $survey; + } + } my $component_args; @@ -251,15 +415,21 @@ # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then # choose the chips containing the center pixel by calling selectComponets() below - if ($component or $use_imfile_id or !$skycenter) { - $command = "$chiptool -processedimfile -pstamp_order -dbname $imagedb"; - if ($component and $component ne 'all') { - $class_id = $component; - $component_args = " -class_id $class_id"; - } - } else { - $command = "$chiptool -listrun -pstamp_order -dbname $imagedb"; + if ($use_releasetool) { + $command = "$releasetool -dbname $imagedb -listrelexp $release_args"; $choose_components = 1; - } - $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id"; + } else { + $command = "$chiptool -pstamp_order -dbname $imagedb"; + if ($component or $use_imfile_id or !$skycenter) { + $command .= " -processedimfile"; + if ($component and $component ne 'all') { + $class_id = $component; + $component_args = " -class_id $class_id"; + } + } else { + $command .= " -listrun"; + $choose_components = 1; + } + $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id"; + } # XXX: perhaps we should stop resolving images to this level here and do it at job run time. # With the mdc file it has all of the information that it needs. @@ -275,16 +445,23 @@ $want_astrom = 1; $set_class_id = 1; - } elsif ($stage eq "warp") { - if ($component or $use_imfile_id or !$skycenter) { - $command = "$warptool -warped -pstamp_order -dbname $imagedb"; - if ($component and $component ne 'all') { - $skycell_id = $component; - $component_args = " -skycell_id $skycell_id"; - } - } else { - $command = "$warptool -listrun -pstamp_order -dbname $imagedb"; + } elsif ($stage eq 'warp') { + if ($use_releasetool) { + $command = "$releasetool -dbname $imagedb -listrelexp $release_args"; + $skycell_id = $component; $choose_components = 1; - } - $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id"; + } else { + $command = "$warptool -pstamp_order -dbname $imagedb"; + if ($component or $use_imfile_id or !$skycenter) { + $command .= " -warped"; + if ($component and $component ne 'all') { + $skycell_id = $component; + $component_args = " -skycell_id $skycell_id"; + } + } else { + $command .= " -listrun"; + $choose_components = 1; + } + $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id"; + } $image_name = "PSWARP.OUTPUT"; $mask_name = "PSWARP.OUTPUT.MASK"; @@ -294,4 +471,5 @@ $base_name = "path_base"; # name of the field for the warptool output } elsif ($stage eq "diff") { + # XXX: is this path ever used for diff images anymore? if ($component or $use_imfile_id or !$skycenter) { $command = "$difftool -diffskyfile -pstamp_order -dbname $imagedb"; @@ -314,8 +492,22 @@ } elsif ($stage eq "stack") { $skycell_id = $component; - $command = "$stacktool -sumskyfile -dbname $imagedb"; - # only consider stacks in full state. - $command .= " -state full"; - $id_opt = "-stack_id"; + if ($use_releasetool) { + $command = "$releasetool -listrelstack -dbname $imagedb $release_args"; + $base_name = 'stack_path_base'; + my $stack_type = $row->{RUN_TYPE}; + if (!isnull($stack_type)) { + if (lc($stack_type) eq 'notnightly') { + $command .= " -stack_type deep -stack_type reference"; + } else { + $command .= " -stack_type $stack_type"; + } + } + } else { + $command = "$stacktool -sumskyfile -dbname $imagedb"; + # only consider stacks in full state. + $command .= " -state full"; + $id_opt = "-stack_id"; + $base_name = 'path_base'; + } $component_args = " -skycell_id $skycell_id" if $skycell_id and $skycell_id ne 'all'; @@ -331,10 +523,9 @@ # this is wrong but gets the right answer. Need to figure out how to find the # rule properly - #$cmf_name = "PSPHOT.OUTPUT"; # this puts .fpa. in the name $cmf_name = "PSWARP.OUTPUT.SOURCES"; + # $cmf_name = "PSPHOT.OUTPUT"; # this puts .fpa. in the name $psf_name = "PPSTACK.TARGET.PSF"; - $base_name = "path_base"; } else { - die "Unknown IMG_TYPe supplied: $stage"; + die "Unknown IMG_TYPE supplied: $stage"; } @@ -371,6 +562,6 @@ $command .= " -dateobs_end $dateobs_end" if $dateobs_end; } elsif ($req_type ne "byid") { - $command .= " -mjd_obs_begin $mjd_min" if $mjd_min; - $command .= " -mjd_obs_end $mjd_max" if $mjd_max; + $command .= " -mjd_min $mjd_min" if $mjd_min; + $command .= " -mjd_max $mjd_max" if $mjd_max; } @@ -379,6 +570,13 @@ if (!isnull($data_group)) { $command .= " -data_group $data_group"; - } elsif ($req_type eq 'byskycell') { + } elsif (!$use_releasetool and $req_type eq 'byskycell') { + # XXX: Why am I using default data_groups only for byskycell requests? $command .= $default_data_groups; + } + + if ($use_releasetool) { + # fwhm arguments are currently only supported when using releasetool for lookups + $command .= " -fwhm_min $fwhm_min" if $fwhm_min; + $command .= " -fwhm_max $fwhm_max" if $fwhm_max; } @@ -394,11 +592,16 @@ $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose); } + if ($choose_components) { # the list of "images" is actually a list of "Runs" # match the coords in the rows to the components in the runs # returns an actual list of images - $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose); + if ($skycenter and $req_type ne 'byskycell') { + $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose); + } else { + $images = selectComponentsByName($ipprc, $imagedb, $req_type, $stage, $component, $rowList, $images, $verbose); + } } else { - # put index for each of the rows into all of the the images + # put index for each of the rows into all of the the selected images setRowRefs($rowList, $images); } @@ -464,5 +667,14 @@ # dquery wants these set $out->{magicked} = 0; - $out->{data_state} = $out->{state}; + if ($use_releasetool) { + if ($image->{skycal_path_base}) { + $out->{astrom} = $image->{skycal_path_base} . ".cmf"; + } + $out->{data_state} = $out->{state} = $out->{stack_state}; + } else { + # XXX: Consider looking up skycal results even if we are + # not using releasetool + $out->{data_state} = $out->{state}; + } } $out->{stage_id} = $stage_id; @@ -472,5 +684,15 @@ my $mask_base; if ($want_astrom) { - if (! defined $out->{astrom}) { + if ($selectedAstrom) { + $out->{astrom} = $selectedAstrom; + $out->{cam_path_base} = $selectedAstrom; + if ($selectedAstrom =~ /\.smf$/) { + $out->{cam_path_base} =~ s/\.smf$//; + } elsif ($selectedAstrom =~ /\.cmf$/) { + $out->{cam_path_base} =~ s/\.cmf$//; + } else { + die ("ERROR: don't know how to extract cam_path_base from $selectedAstrom\n"); + } + } elsif (! defined $out->{astrom}) { if (! find_astrometry($ipprc, $imagedb, $out, $verbose)) { print STDERR "failed to find astrometry for $stage $stage_id\n"; @@ -507,5 +729,5 @@ my $byid = shift; my $option_mask = shift; - my $img_type = shift; + my $stage = shift; my $verbose = shift; @@ -558,4 +780,6 @@ } + # XXX: The logic below is somewhat messed up. See XXX: ... below + # # The standard way to do a diff is warp - stack # so we interpret the requested image in that way @@ -572,9 +796,14 @@ } my $stack_id; + if ($image->{diff_mode} == 2) { + # XXX: .. stack_id wasn't getting set + $stack_id = $stack2; + $image->{stack_id} = $stack_id; + } if ($stack1 and $stack2) { # we have a stack - stack diff (well it might be stack - warp....) # but at any rate we only handle image type diff and stack - if (($img_type ne "diff") and ($img_type ne "stack")) { - print STDERR "lookup_diff: cannot lookup IMG_TYPE $img_type bydiff from a stack stack diff run\n"; + if (($stage ne "diff") and ($stage ne "stack")) { + print STDERR "lookup_diff: cannot lookup IMG_TYPE $stage bydiff from a stack stack diff run\n"; setErrorCodes($rowList, $PSTAMP_INVALID_REQUEST); # all images will be the same so we can stop @@ -627,5 +856,5 @@ } - if ($img_type eq "diff") { + if ($stage eq "diff") { my @imageList = ($image); @@ -658,7 +887,7 @@ sub lookup_bycoord { my $ipprc = shift; - my $rowList = shift; + my $row = shift; my $imagedb = shift; - my $img_type = shift; + my $stage = shift; my $tess_id = shift; my $component = shift; @@ -666,18 +895,30 @@ my $ra = shift; my $dec = shift; - my $dateobs_begin = shift; + my $dateobs_begin = shift; # these are used for single frame lookups my $dateobs_end = shift; my $filter = shift; my $data_group = shift; my $option_mask = shift; - my $mjd_min = shift; - my $mjd_max = shift; + my $mjd_min = shift; # these are used for stack lookups + my $mjd_max = shift; # these are used for stack lookups my $verbose = shift; + my $fwhm_min = getValOrZero($row->{FWHM_MIN}); + my $fwhm_max = getValOrZero($row->{FWHM_MAX}); + + + my ($release_name, $survey, $default_tess_id) = get_release_info($row); + if (!$tess_id and $default_tess_id) { + $tess_id = $default_tess_id; + } + + my $rowList = [$row]; + my $results = (); - if (($img_type eq "raw") or ($img_type eq "chip")) { - - my $chips = lookup_by_cam_id_and_coords($ipprc, $imagedb, $img_type, - $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose); + if (($stage eq "raw") or ($stage eq "chip")) { + + my $chips = lookup_by_cam_id_and_coords($ipprc, $imagedb, $stage, + $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose, + $release_name, $survey, $fwhm_min, $fwhm_max); if (!$chips or scalar @$chips == 0) { @@ -686,7 +927,10 @@ foreach my $chip (@$chips) { next if $component and ($chip->{component} ne $component); - my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $chip->{id}, + my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $stage, $chip->{id}, $tess_id, $chip->{component}, $need_magic, - $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); + $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, + $fwhm_min, $fwhm_max, + $chip->{astrom}, undef, undef, + $verbose); next if !$these_results; @@ -697,6 +941,6 @@ } else { # this should have been checked elsewhere - die "unexpected image type $img_type" if ($img_type ne "warp") - and ($img_type ne "stack") and ($img_type ne "diff"); + die "unexpected image type $stage" if ($stage ne "warp") + and ($stage ne "stack") and ($stage ne "diff"); my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose); @@ -705,8 +949,12 @@ setErrorCodes($rowList, $PSTAMP_NO_IMAGE_MATCH); } else { + # XXX: We are not applying the fwhm cuts foreach my $skycell (@$skycells) { - my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef, + my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef, $skycell->{tess_id}, $skycell->{component}, $need_magic, - $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); + $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, + $fwhm_min, $fwhm_max, + undef, $release_name, $survey, + $verbose); next if !$these_results; @@ -724,10 +972,10 @@ sub lookup_by_cam_id_and_coords { - my $ipprc = shift; - my $imagedb = shift; - my $img_type = shift; - my $ra = shift; - my $dec = shift; - my $need_magic = shift; + my $ipprc = shift; + my $imagedb = shift; + my $stage = shift; + my $ra = shift; + my $dec = shift; + my $need_magic = shift; my $dateobs_begin = shift; my $dateobs_end = shift; @@ -735,15 +983,32 @@ my $data_group = shift; my $verbose = shift; - - my $camruns; - - { - my $search_radius = 1.6; # XXX: this should be camera specific - + my $release_name = shift; + my $survey = shift; + my $fwhm_min = shift; + my $fwhm_max = shift; + + my $search_radius = 1.6; # XXX: this should be camera specific + + my $command; + my $using_camtool = 0; + if ($release_name or $survey) { + $command = "$releasetool -dbname $imagedb -listrelexp"; + if ($survey) { + $command .= " -surveyName $survey"; + } + if ($release_name) { + $command .= " -release_name $release_name"; + } else { + $command .= " -priority_order"; + } + if ($fwhm_min != 0) { + $command .= " -fwhm_min $fwhm_min"; + } + if ($fwhm_max != 0) { + $command .= " -fwhm_max $fwhm_max"; + } + } else { + $using_camtool = 1; my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order"; - $command .= " -ra $ra -decl $dec -radius $search_radius"; - $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; - $command .= " -dateobs_end $dateobs_end" if $dateobs_end ; - $command .= " -filter $filter" if $filter; # NOTE: we are applying the data_group to the camera run. # If we're looking for chip stage images there is no guarentee that @@ -757,28 +1022,37 @@ } $command .= " -destreaked" if $need_magic; - - # run the tool and parse the output - $camruns = runToolAndParse($command, $verbose); - } - if (!$camruns) { + } + $command .= " -ra $ra -decl $dec -radius $search_radius"; + $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; + $command .= " -dateobs_end $dateobs_end" if $dateobs_end ; + $command .= " -filter $filter" if $filter; + + # run the tool and parse the output + my $results = runToolAndParse($command, $verbose); + if (!$results) { return undef; } my $components; my $last_exp_id = 0; - foreach my $camRun (@$camruns) { - my $cam_id = $camRun->{cam_id}; - my $exp_id = $camRun->{exp_id}; - + foreach my $result (@$results) { + my $cam_id = $result->{cam_id}; + my $exp_id = $result->{exp_id}; + + # go on if we already have a result for this exposure next if $exp_id eq $last_exp_id; - next if $camRun->{quality}; - next if $camRun->{fault}; - - updateCamRunCache($camRun); + next if $result->{quality}; + + if ($using_camtool) { + next if $result->{fault}; + } else { + $result->{path_base} = $result->{cam_path_base}; + } + updateCamRunCache($result); $last_exp_id = $exp_id; # XXX Use file rule - my $astrom = $camRun->{path_base} . ".smf"; + my $astrom = $result->{path_base} . ".smf"; my $astrom_resolved = $ipprc->file_resolve($astrom); next if !$astrom_resolved; @@ -834,5 +1108,5 @@ } - print "cam_id $cam_id exp_id $camRun->{exp_id} chip_id $camRun->{chip_id} $class_id\n"; + print "cam_id $cam_id exp_id $result->{exp_id} chip_id $result->{chip_id} $class_id\n"; # build the hash to return @@ -842,19 +1116,20 @@ # problem my $comp = { - exp_id => $camRun->{exp_id}, - exp_name => $camRun->{exp_name}, - chip_id => $camRun->{chip_id}, - cam_id => $camRun->{cam_id}, + exp_id => $result->{exp_id}, + exp_name => $result->{exp_name}, + chip_id => $result->{chip_id}, + cam_id => $result->{cam_id}, class_id => $class_id, - component => $class_id + component => $class_id, + astrom => $astrom }; - if ($img_type eq "chip") { - $comp->{id} = $camRun->{chip_id}; - $comp->{state} = $camRun->{chip_state}; - $comp->{magicked} = $camRun->{chip_magicked}; - } else { - $comp->{id} = $camRun->{exp_id}; + if ($stage eq "chip") { + $comp->{id} = $result->{chip_id}; + $comp->{state} = $result->{chip_state}; + $comp->{magicked} = 0; + } else { + $comp->{id} = $result->{exp_id}; $comp->{state} = 'full'; - $comp->{magicked} = $camRun->{raw_magicked}; + $comp->{magicked} = $0; } push @$components, $comp; @@ -1072,4 +1347,9 @@ my $val = shift; return (!defined($val) or ($val == 0)); +} + +sub getValOrZero { + my $val = shift; + return iszero($val) ? 0.0 : $val; } @@ -1283,4 +1563,68 @@ } +sub selectComponentsByName { + my $ipprc = shift; + my $imagedb = shift; + my $req_type = shift; + my $stage = shift; + my $component = shift; + my $rowList = shift; + my $runList = shift; + my $verbose = shift; + my $results = []; + + $component = "" if $component and lc($component) eq 'all'; + + # all runs match all rows so they can share a common row_index + # XXX: I'm not sure we ever actually get here with more than one + # row + my $numRows = scalar @$rowList; + my $row_index = []; + for (my $i=0; $i < $numRows; $i++) { + push @$row_index, $i; + } + + # XXX: Why did I make this restriction? I added byskycell + if (($req_type eq "byid") or ($req_type eq "byexp") or ($req_type eq 'byskycell')) { + my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", ""); + foreach my $run (@$runList) { + # now find the images for this run + #foreach my $c (keys %components) { + my $command; + my $stage_id; + if ($stage eq 'raw') { + $stage_id = $run->{exp_id}; + $command = "$regtool -processedimfile -exp_id $stage_id"; + $command .= " -class_id $component" if $component; + } elsif ($stage eq 'chip') { + $stage_id = $run->{chip_id}; + $command = "$chiptool -processedimfile -chip_id $stage_id"; + $command .= " -class_id $component" if $component; + } elsif ($stage eq 'warp') { + $stage_id = $run->{warp_id}; + $command = "$warptool -warped -warp_id $stage_id"; + $command .= " -skycell_id $component" if $component; + } + $command .= " -dbname $imagedb"; + my $images = runToolAndParse($command, $verbose); + if (!defined $images) { + print "No $component found for $stage ${stage}_id $stage_id\n"; + next; + } + foreach my $image (@$images) { + $image->{row_index} = $row_index; + if (($stage eq "raw") or ($stage eq 'chip')) { + $image->{cam_id} = $run->{cam_id}; + $image->{cam_path_base} = $run->{cam_path_base}; + } + push @$results, $image; + } + } + } else { + &my_die ("selectComponentsByName not supported for REQ_TYPE: $req_type\n", $PS_EXIT_PROG_ERROR); + } + return $results; +} + sub filterRuns { my $stage = shift; @@ -1409,4 +1753,6 @@ $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); + $releasetool = can_run('releasetool') or (warn "Can't find releasetool" + and $missing_tools = 1); $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1); $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); @@ -1464,4 +1810,30 @@ } +sub get_release_info { + my $row = shift; + + my $survey; + my $release_name = $row->{IPP_RELEASE}; + if (isnull($release_name)) { + $release_name = ""; + # no release check for survey + $survey = isnull($row->{SURVEY_NAME}) ? "" : $row->{SURVEY_NAME}; + } else { + # if release is given leave survey empty + $survey = ""; + } + + my $tess_id; + if ($release_name) { + # hack to optimize bycoord and byskycell lookups for well known releases + # XXX: todo query database and return a list for the survey or release + if ($release_name eq '3PI.PV1') { + $tess_id = 'RINGS.V3'; + } + } + + return ($release_name, $survey, $tess_id); +} + sub my_die { Index: /tags/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- /tags/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 35488) +++ /tags/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 35489) @@ -44,4 +44,5 @@ $PSTAMP_DUP_REQUEST $PSTAMP_INVALID_REQUEST + $PSTAMP_UNKNOWN_PROJECT $PSTAMP_UNKNOWN_PRODUCT $PSTAMP_NO_IMAGE_MATCH @@ -80,4 +81,6 @@ our $PSTAMP_USE_IMFILE_ID = 16384; our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768; + +# these bits will be repurposed our $PSTAMP_REQUEST_UNCENSORED = 0x10000; our $PSTAMP_REQUIRE_UNCENSORED = 0x20000; @@ -93,9 +96,10 @@ our $PSTAMP_DUP_REQUEST = 20; our $PSTAMP_INVALID_REQUEST = 21; -our $PSTAMP_UNKNOWN_PRODUCT = 22; +our $PSTAMP_UNKNOWN_PROJECT = 22; +our $PSTAMP_UNKNOWN_PRODUCT = 22; #this error code was mis-named it is left for compatabiliyt our $PSTAMP_NO_IMAGE_MATCH = 23; our $PSTAMP_NOT_DESTREAKED = 24; our $PSTAMP_NOT_AVAILABLE = 25; -our $PSTAMP_GONE = 26; +our $PSTAMP_GONE = 26; # this value is used in ippTools our $PSTAMP_NO_JOBS_QUEUED = 27; our $PSTAMP_NO_OVERLAP = 28; @@ -131,5 +135,5 @@ PSTAMP_DUP_REQUEST PSTAMP_INVALID_REQUEST -PSTAMP_UNKNOWN_PRODUCT +PSTAMP_UNKNOWN_PROJECT PSTAMP_NO_IMAGE_MATCH PSTAMP_NOT_DESTREAKED @@ -139,4 +143,5 @@ PSTAMP_NO_OVERLAP PSTAMP_NOT_AUTHORIZED +PSTAMP_NOT_AUTHORIZED ); @@ -167,5 +172,5 @@ my $fields_output; { - my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USERNAME EMAIL"; + my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION EMAIL"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -187,16 +192,12 @@ if ($extver > 1) { $header{ACTION} = $action; - $header{USER} = $username; $header{EMAIL} = $email; } else { $header{ACTION} = $action = "PROCESS"; - $header{USERNAME} = 'null'; $header{EMAIL} = 'null'; } - if ($action eq "LIST") { - return (\%header, undef); - } elsif ($action ne "PROCESS") { - die "unexpected request ACTION found: $action in $request_file_name"; + if ($action ne "PROCESS" and $action ne 'PREVIEW') { + die "\nunexpected request ACTION found: $action in $request_file_name"; } Index: /tags/ipp-20130307/dbconfig/changes.txt =================================================================== --- /tags/ipp-20130307/dbconfig/changes.txt (revision 35488) +++ /tags/ipp-20130307/dbconfig/changes.txt (revision 35489) @@ -2388,6 +2388,9 @@ CREATE TABLE lapGroup ( seq_id BIGINT, + tess_id VARCHAR(64), projection_cell VARCHAR(64), state VARCHAR(16), + label VARCHAR(654), + registered TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP fault SMALLINT, PRIMARY KEY(seq_id, projection_cell), @@ -2397,15 +2400,19 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; -UPDATE dbversion set schema_version = '1.1.75', updated= CURRENT_TIMESTAMP(); - --- new Postage Stamp Request columns (NOTE these changes to not apply to gpc1 --- because the pstamp tables have been dropped from that database +-- new Postage Stamp Request columns +-- Note: These do not aply to the gpc1 database because the pstamp tables were +-- deleted from that database. ALTER TABLE pstampRequest ADD COLUMN username VARCHAR(255) AFTER outdir; ALTER TABLE pstampRequest ADD COLUMN proj_id BIGINT AFTER username; -ALTER TABLE pstampRequest ADD COLUMN registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER proj_id; - --- default to the gpc1 project (or is it ps1?) +ALTER TABLE pstampRequest ADD COLUMN registered DATETIME AFTER proj_id; + +-- default to the gpc1 project UPDATE pstampRequest SET proj_id = 1; - - +-- unique keys for the release tables + +alter table relStack add unique key (rel_id, tess_id, skycell_id, filter, stack_type, mjd_obs) +alter table relExp add UNIQUE KEY(rel_id, exp_id); +alter table ippRelease add unique key (surveyID, release_name); + +UPDATE dbversion set schema_version = '1.1.75', updated= CURRENT_TIMESTAMP(); Index: /tags/ipp-20130307/dbconfig/lap.md =================================================================== --- /tags/ipp-20130307/dbconfig/lap.md (revision 35488) +++ /tags/ipp-20130307/dbconfig/lap.md (revision 35489) @@ -33,6 +33,9 @@ lapGroup METADATA seq_id S64 0 + tess_id STR 64 projection_cell STR 64 state STR 64 + label STR 64 + registered TAI NULL fault S16 0 end Index: /tags/ipp-20130307/ippScripts/Build.PL =================================================================== --- /tags/ipp-20130307/ippScripts/Build.PL (revision 35488) +++ /tags/ipp-20130307/ippScripts/Build.PL (revision 35489) @@ -130,4 +130,5 @@ scripts/regenerate_background.pl scripts/relgroup_exp_list.pl + scripts/queuestaticsky.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: /tags/ipp-20130307/ippScripts/scripts/camera_exp.pl =================================================================== --- /tags/ipp-20130307/ippScripts/scripts/camera_exp.pl (revision 35488) +++ /tags/ipp-20130307/ippScripts/scripts/camera_exp.pl (revision 35489) @@ -363,5 +363,7 @@ } # Construct FPA continuity corrected background images - { + if ($camera =~ /ISP/) { + print "Skipping FPA continuity corrected background images for ISP\n"; + } else { my $command; $command = "$ppImage"; Index: /tags/ipp-20130307/ippScripts/scripts/mergedvodb_merge.pl =================================================================== --- /tags/ipp-20130307/ippScripts/scripts/mergedvodb_merge.pl (revision 35488) +++ /tags/ipp-20130307/ippScripts/scripts/mergedvodb_merge.pl (revision 35489) @@ -156,5 +156,5 @@ $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path"; } else { - $merge_command = "$dvomerge $minidvodb_path into $mergedvodb_path"; + $merge_command = "$dvomerge -parallel $minidvodb_path into $mergedvodb_path"; } print "\n$merge_command\n"; Index: /tags/ipp-20130307/ippScripts/scripts/minidvodb_premerge.pl =================================================================== --- /tags/ipp-20130307/ippScripts/scripts/minidvodb_premerge.pl (revision 35488) +++ /tags/ipp-20130307/ippScripts/scripts/minidvodb_premerge.pl (revision 35489) @@ -16,4 +16,5 @@ my $dtime_relphot; my $dtime_script; +my $dtime_delstar; use vars qw( $VERSION ); @@ -33,7 +34,9 @@ my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $delstar = can_run('delstar') or (warn "Can't find delstar" and $missing_tools = 1); my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); + my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1); @@ -80,5 +83,26 @@ unless ($no_op) { - #this is chopped into several parts: addstar, relphot + #this is chopped into several parts: delstar,addstar, relphot, relastro, dvoverify + #delstar - first step: are there duplicates, if so remove them + { + my $command = "$delstar -update -dup-images "; + #$command .= " -D CAMERA $camera"; + $command .= " -D CATDIR $minidvodb"; + my $mjd_delstar_start = DateTime->now->mjd; # MJD of starting script + print "\n$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform delstar: $error_code", $minidvodb_id, $error_code); + } + print $full_buf; + $dtime_delstar = 86400.0*(DateTime->now->mjd - $mjd_delstar_start); + # MJD of starting script + print "delstar time $dtime_delstar\n"; + } + + + #addstar { @@ -94,6 +118,6 @@ &my_die("Unable to perform addstar: $error_code", $minidvodb_id, $error_code); } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); $dtime_resort = $dtime_addstar; - # MJD of starting script + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); + # MJD of starting script $dtime_resort = $dtime_addstar; print "addstar -resort time $dtime_addstar\n"; @@ -105,7 +129,9 @@ # relphot only takes lower case gpc1 my $relphot_camera = lc($camera); - my $command = "$relphot -averages -update"; - $command .= " -D CAMERA $relphot_camera"; - $command .= " -D CATDIR $minidvodb"; + #my $command = "$relphot -averages -update"; + #$command .= " -D CAMERA $relphot_camera"; + #$command .= " -D CATDIR $minidvodb"; + #this is a friday hack + my $command = "echo skipping relphot"; print "$command\n"; my $mjd_relphot_start = DateTime->now->mjd; # MJD of starting script Index: /tags/ipp-20130307/ippScripts/scripts/queuestaticsky.pl =================================================================== --- /tags/ipp-20130307/ippScripts/scripts/queuestaticsky.pl (revision 35489) +++ /tags/ipp-20130307/ippScripts/scripts/queuestaticsky.pl (revision 35489) @@ -0,0 +1,273 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use POSIX qw(strftime); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my ($seq_id, $tess_id, $projection_cell, $label, $dist_group, $dbname, $pretend, $simple, $verbose, $no_update); + +GetOptions( + 'seq_id=s' => \$seq_id, + 'tess_id=s' => \$tess_id, + 'projection_cell=s' => \$projection_cell, + 'label=s' => \$label, + 'dist_group=s' => \$dist_group, + 'dbname=s' => \$dbname, + 'pretend' => \$pretend, + 'simple' => \$simple, + 'no-update' => \$no_update, + 'verbose|v' => \$verbose, +) or pod2usage(2); + +unless (defined $seq_id and defined $tess_id and defined $projection_cell and defined $label and defined $dist_group) { + warn ("label, seq_id, tess_id, and projection_cell are required\n"); + pod2usage(2); +}; + + +my $missing_tools; +my $laptool = can_run('laptool') or (warn "Can't find laptool" and $missing_tools = 1); +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $PS_EXIT_CONFIG_ERROR ); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# XXX get from site.config +my $workdirBase = "neb://\@HOST\@.0/gpc1"; + +my @filters; +{ + my $command = "$laptool -filtersforgroup"; + $command .= " -seq_id $seq_id"; + $command .= " -tess_id $tess_id"; + $command .= " -projection_cell $projection_cell"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => 0); + unless ($success) { + $error_code = $error_code >> 8; + my_die("failed to run $command $error_code\n", $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); + my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + + foreach my $entry (@$list) { + push @filters, $entry->{filter}; + } +} + +my $nFilters = scalar @filters; +print STDERR "lapGroup $seq_id $tess_id $projection_cell has $nFilters filters\n"; + +&my_die("No filters found for $seq_id $tess_id $projection_cell", $PS_EXIT_PROG_ERROR) unless $nFilters; +&my_die("Unexpected number of filters found for $seq_id $tess_id $projection_cell", $PS_EXIT_PROG_ERROR) unless $nFilters <= 5; + +my $datestr = strftime "%Y/%m/%d", gmtime; + +my $staticsky_command="staticskytool -definebyquery" + . " -select_label $label" + . " -select_tess_id $tess_id" + . " -select_skycell_id $projection_cell%" + . " -set_workdir $workdirBase/$label/$datestr" + . " -set_label $label" + . " -set_data_group $label" + . " -set_dist_group $dist_group"; + +$staticsky_command .= " -pretend" if $pretend; +$staticsky_command .= " -simple" if $simple; +$staticsky_command .= " -dbname $dbname" if $dbname; + +for (my $num = $nFilters; $num > 0; $num--) { + + # set up the possible combinations of $num filters from @filters + + my $filter_combos = setup_filter_combos($num); + + # printcombos($filter_combos) if $verbose; + + foreach my $filter_combo (@$filter_combos) { + my $command = $staticsky_command; + foreach my $f (@$filter_combo) { + $command .= " -select_filter $f"; + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = $error_code >> 8; + my_die("failed to run $command $error_code\n", $error_code); + } + } +} +{ + my $command = "$laptool -updategroup"; + $command .= " -seq_id $seq_id"; + $command .= " -tess_id $tess_id"; + $command .= " -projection_cell $projection_cell"; + $command .= " -set_state full"; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = $error_code >> 8; + carp("failed to run $command $error_code\n", $error_code); + exit $error_code; + } + } else { + print STDERR "Skipping $command\n"; + } +} + +exit 0; + +sub my_die +{ + my $msg = shift; + my $exit_code = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code; + + carp($msg); + + my $command = "$laptool -updategroup"; + $command .= " -seq_id $seq_id"; + $command .= " -tess_id $tess_id"; + $command .= " -projection_cell $projection_cell"; + $command .= " -set_fault $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = $error_code >> 8; + carp("failed to run $command $error_code\n", $error_code); + exit $error_code; + } + } else { + print STDERR "Skipping $command\n"; + } + exit $exit_code; +} + + +# The rest of this file contains some subroutines for setting up the filter combinations + + +sub setup_filter_combos { + my $combos = shift; + + my_die( "invalid combos arg: $combos", $PS_EXIT_PROG_ERROR) + unless defined $combos and ($combos eq "5" or $combos eq "4" or $combos eq "3" or $combos eq "2" or $combos eq "1"); + + if ($combos eq 5) { + my @ary; + push @ary, \@filters; + return \@ary; + } elsif ($combos eq 4) { + return makecombo4(); + } elsif ($combos eq 3) { + return makecombo3(); + } elsif ($combos eq 2) { + return makecombo2(); + } elsif ($combos eq 1) { + my @ary; + foreach my $f (@filters) { + my @ary2 = ($f); + push @ary, \@ary2 + } + return \@ary; + } else { + die "how did we get here?"; + } +} + + + +sub printcombos { + my $combos = shift; + foreach my $c (@$combos) { + my $str; + foreach my $f (@$c) { + $str .= " -select_filter $f"; + } + print "$str\n"; + } +} + + +sub makecombo4 { + my $numFilters = scalar @filters; + my @combos; + + for (my $i = 0; $i < $numFilters; $i++) { + for (my $j = $i + 1; $j < $numFilters; $j++) { + for (my $k = $j + 1; $k < $numFilters; $k++) { + for (my $l = $k + 1; $l < $numFilters; $l++) { + my @combo = ($filters[$i], $filters[$j], $filters[$k], $filters[$l]); + push @combos, \@combo; + } + } + } + } + return \@combos; +} + +sub makecombo3 { + my $numFilters = scalar @filters; + my @combos; + + for (my $i = 0; $i < $numFilters; $i++) { + for (my $j = $i + 1; $j < $numFilters; $j++) { + for (my $k = $j + 1; $k < $numFilters; $k++) { + my @combo = ($filters[$i], $filters[$j], $filters[$k]); + push @combos, \@combo; + } + } + } + return \@combos; +} + +sub makecombo2 { + my $numFilters = scalar @filters; + my @combos; + + for (my $i = 0; $i < $numFilters; $i++) { + for (my $j = $i + 1; $j < $numFilters; $j++) { + my @combo = ($filters[$i], $filters[$j]); + push @combos, \@combo; + } + } + return \@combos; +} + + +__END__ + +=pod + +=head1 NAME + +queuesskylap - queue LAP staticsky runs + +=head1 SYNOPSIS + + XXX: pod TODO + + queuesskylap --ra_min --ra_max --dec_min --dec_max [--go] + + Index: /tags/ipp-20130307/ippScripts/scripts/relgroup_exp_list.pl =================================================================== --- /tags/ipp-20130307/ippScripts/scripts/relgroup_exp_list.pl (revision 35488) +++ /tags/ipp-20130307/ippScripts/scripts/relgroup_exp_list.pl (revision 35489) @@ -16,10 +16,11 @@ -my ($group_id, $group_type, $lap_id, $release_name, $rebuild, $dbname, $no_update, $verbose); +my ($group_id, $group_type, $lap_id, $group_name, $release_name, $rebuild, $dbname, $no_update, $verbose); GetOptions( 'group_id=s' => \$group_id, + 'group_type=s' => \$group_type, 'lap_id=s' => \$lap_id, - 'group_type=s' => \$group_type, + 'group_name=s' => \$group_name, 'release_name=s' => \$release_name, 'rebuild' => \$rebuild, # rebuild directory if it exists @@ -36,4 +37,9 @@ -exitval => 3) unless $lap_id; + $group_name = "lap_$lap_id"; +} else { + pod2usage( -msg => "--group_name is required with --group_type $group_type", + -exitval => 3) + unless $group_name; } @@ -48,52 +54,26 @@ # XXX: get this from site.config -my $outputBase = "/data/ippc17.0/www/ipp-misc/ps1-data/single-frame/$release_name"; - -my $outdir; -if ($group_type eq 'lap') { - $outdir = "$outputBase/lap_$lap_id"; - if (-e $outdir) { - if ($rebuild) { - my $rc = system "rm -rf $outdir"; - if ($rc) { - my $status = $rc >> 8; - my_die("failed to remove existing directory $outdir: $rc $status", $group_id, $PS_EXIT_UNKNOWN_ERROR); - } - } else { - my_die("directory $outdir already exists and --rebuild not supplied", $group_id, $PS_EXIT_UNKNOWN_ERROR); - } - } -} else { - my_die("Not ready to do group_type $group_type\n", $group_id, $PS_EXIT_UNKNOWN_ERROR); +my $exposureListRoot = "/data/ippc17.0/www/ipp-misc/ps1-data/single-frame"; + +my $outputBase = "$exposureListRoot/$release_name"; + +if (!-e $outputBase) { + mkdir $outputBase or &my_die("Failed to create $outputBase", $group_id, $PS_EXIT_SYS_ERROR); +} + +my $outdir = "$outputBase/$group_name"; +if (-e $outdir) { + if ($rebuild) { + my $rc = system "rm -rf $outdir"; + if ($rc) { + my $status = $rc >> 8; + my_die("failed to remove existing directory $outdir: $rc $status", $group_id, $PS_EXIT_UNKNOWN_ERROR); + } + } else { + my_die("directory $outdir already exists and --rebuild not supplied", $group_id, $PS_EXIT_UNKNOWN_ERROR); + } } mkdir $outdir or my_die ("failed to create $outdir", $group_id, $PS_EXIT_UNKNOWN_ERROR); - -if (0) { - # XXX: old code from prototype for nightly lists - my $outroot; - my $oh; - my ($year, $month, $day); - if ($outroot) { - die "usage: $0 \n" unless $year and $month and $day; - my $monthdir = sprintf "$outroot/%4d/%02d", $year, $month; - if (! -e $monthdir) { - die "can't find directory for $year-$month\n"; - } - my $daydir = sprintf "$monthdir/%02d", $day; - if (! -e $daydir) { - mkdir $daydir or die "failed to create $daydir\n"; - } - my $outfile = sprintf "$daydir/index.html", $day; - open $oh, ">$outfile" or die "failed to open $outfile"; - } else { - $oh = *STDOUT; - } - - my $ticks = timegm(0, 0, 0, $day, $month - 1, $year - 1900); - - my $mjd = 40587 + ($ticks / 86400.); - my $date = sprintf "%04d-%02d-%02d", $year, $month, $day; -} my $oh; @@ -105,5 +85,10 @@ open $oh2, ">$outfile2" or my_die("failed to open $outfile2", $group_id, $PS_EXIT_UNKNOWN_ERROR); -my $title = "PS1 Exposure List for $release_name LAP run $lap_id"; +my $title = "PS1 Exposure List for"; +if ($group_type eq 'lap') { + $title .= " $release_name LAP run $lap_id"; +} else { + $title .= " $group_name"; +} my $head = " @@ -125,5 +110,5 @@ -# find all exposures on given day that have a chip run +# find all exposures in this relGroup my $query = " SELECT @@ -149,5 +134,5 @@ $stmt->execute() or my_die("failed to execute query: $stmt->errstr\n", $group_id, $PS_EXIT_UNKNOWN_ERROR); -printf STDERR "%4d exposures in group for LAP run $lap_id\n", $stmt->rows; +printf STDERR "%4d exposures in group for $group_name\n", $stmt->rows; # $stmt->dump_results(1000); Index: /tags/ipp-20130307/ippScripts/scripts/whichimage =================================================================== --- /tags/ipp-20130307/ippScripts/scripts/whichimage (revision 35488) +++ /tags/ipp-20130307/ippScripts/scripts/whichimage (revision 35489) @@ -94,4 +94,6 @@ my $cmd = "dvoImagesAtCoords -chipcoords $coord_file"; $cmd .= " -listchipcoords" if $listchipcoords; + +$tess_id = uc($tess_id) if $tess_id; my $status = 0; @@ -125,4 +127,5 @@ # next if $tess_name eq 'FIXNS'; # next if $tess_name eq 'ALLSKY'; + next if $tess_name eq 'RINGS.V0'; push @tess_ids_to_check, $tess_name; } Index: /tags/ipp-20130307/ippTasks/lapgroup.pro =================================================================== --- /tags/ipp-20130307/ippTasks/lapgroup.pro (revision 35489) +++ /tags/ipp-20130307/ippTasks/lapgroup.pro (revision 35489) @@ -0,0 +1,161 @@ +## lapgroup.pro : tasks for lap group management : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/lapgroup +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init pendinglapGroup + +### Database lists +$lapgroup_DB = 0 + +### Check status of lapgroup tasks +macro lapgroup.status + book listbook pendinglapGroup +end + +### Reset lapgroup tasks +macro lapgroup.reset + book init pendinglapGroup +end + +### Turn lapgroup tasks on +macro lapgroup.on + task lapgroup.load + active true + end + task lapgroup.run + active true + end +end + +### Turn lapgroup tasks off +macro lapgroup.off + task lapgroup.load + active false + end + task lapgroup.run + active false + end +end + +### Load jobs for lapGroup +### Tasks are loaded into pendinglapGroup. +task lapgroup.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/lapgroup.log + + task.exec + if ($LABEL:n == 0) break + $run = laptool -pendinggroup + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$lapgroup_DB + $run = $run -dbname $DB:$lapgroup_DB + $lapgroup_DB ++ + if ($lapgroup_DB >= $DB:n) set lapgroup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pendinglapGroup -key seq_id:tess_id:projection_cell -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pendinglapGroup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pendinglapGroup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task lapgroup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 120 + npending 1 + + task.exec + book npages pendinglapGroup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in pendinglapGroup (pantaskState == INIT) + book getpage pendinglapGroup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pendinglapGroup $pageName pantaskState RUN + book getword pendinglapGroup $pageName seq_id -var SEQ_ID + book getword pendinglapGroup $pageName tess_id -var TESS_ID + book getword pendinglapGroup $pageName projection_cell -var PROJECTION_CELL + book getword pendinglapGroup $pageName label -var LABEL + book getword pendinglapGroup $pageName dist_group -var DIST_GROUP + book getword pendinglapGroup $pageName dbname -var DBNAME + + stdout $LOGSUBDIR/lapgroup.log + stderr $LOGSUBDIR/lapgroup.log + + host anyhost + + $run = queuestaticsky.pl --seq_id $SEQ_ID --tess_id $TESS_ID --projection_cell $PROJECTION_CELL --label $LABEL --dist_group $DIST_GROUP + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pendinglapGroup $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword pendinglapGroup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pendinglapGroup $options:0 pantaskState TIMEOUT + end +end + Index: /tags/ipp-20130307/ippTasks/pstamp.pro =================================================================== --- /tags/ipp-20130307/ippTasks/pstamp.pro (revision 35488) +++ /tags/ipp-20130307/ippTasks/pstamp.pro (revision 35489) @@ -965,5 +965,5 @@ # XXX: have the script set this up this - $MYLOGFILE="/data/ippc17.0/pstamp/work/logs/cleanup.$REQ_ID" + $MYLOGFILE=/data/ippc17.0/pstamp/work/logs/cleanup.$REQ_ID stdout $MYLOGFILE stderr $MYLOGFILE Index: /tags/ipp-20130307/ippTasks/release.pro =================================================================== --- /tags/ipp-20130307/ippTasks/release.pro (revision 35489) +++ /tags/ipp-20130307/ippTasks/release.pro (revision 35489) @@ -0,0 +1,167 @@ +## release.pro : tasks for release management : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/release +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init pendingrelGroup + +### Database lists +$relgroup_DB = 0 + +### Check status of release tasks +macro release.status + book listbook pendingrelGroup +end + +### Reset release tasks +macro release.reset + book init pendingrelgroup +end + +### Turn release tasks on +macro release.on + task relgroup.load + active true + end + task relgroup.run + active true + end +end + +### Turn release tasks off +macro release.off + task relgroup.load + active false + end + task relgroup.run + active false + end +end + +### Load jobs for relGroup +### Tasks are loaded into pendingrelGroup. +task relgroup.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/relgroup.log + + task.exec + if ($LABEL:n == 0) break + $run = releasetool -pendingrelgroup + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$relgroup_DB + $run = $run -dbname $DB:$relgroup_DB + $relgroup_DB ++ + if ($relgroup_DB >= $DB:n) set relgroup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pendingrelGroup -key group_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pendingrelGroup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pendingrelGroup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task relgroup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 120 + npending 1 + + task.exec + book npages pendingrelGroup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in pendingrelGroup (pantaskState == INIT) + book getpage pendingrelGroup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pendingrelGroup $pageName pantaskState RUN + book getword pendingrelGroup $pageName group_id -var GROUP_ID + book getword pendingrelGroup $pageName rel_id -var REL_ID + book getword pendingrelGroup $pageName group_type -var GROUP_TYPE + book getword pendingrelGroup $pageName lap_id -var LAP_ID + book getword pendingrelGroup $pageName group_name -var GROUP_NAME + book getword pendingrelGroup $pageName release_name -var RELEASE_NAME + book getword pendingrelGroup $pageName dbname -var DBNAME + + stdout $LOGSUBDIR/relgroup.log + stderr $LOGSUBDIR/relgroup.log + + host anyhost + + $run = relgroup_exp_list.pl --group_id $GROUP_ID --group_type $GROUP_TYPE --release_name $RELEASE_NAME --rebuild + if ("$GROUP_TYPE" == "lap") + $run = $run --lap_id $LAP_ID + else + $run = $run --group_name $GROUP_NAME + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pendingrelGroup $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword pendingrelGroup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pendingrelGroup $options:0 pantaskState TIMEOUT + end +end + Index: /tags/ipp-20130307/ippTasks/survey.pro =================================================================== --- /tags/ipp-20130307/ippTasks/survey.pro (revision 35488) +++ /tags/ipp-20130307/ippTasks/survey.pro (revision 35489) @@ -18,4 +18,6 @@ book create SURVEY_STATICSKYSINGLE book create SURVEY_SKYCAL + book create SURVEY_LAPGROUP + book create SURVEY_RELEXP $haveSurveyBooks = TRUE end @@ -33,4 +35,6 @@ $SURVEY_PUBLISH_DB = 0 $SURVEY_SKYCAL_DB = 0 +$SURVEY_LAPGROUP_DB = 0 +$SURVEY_RELEXP_DB = 0 $SURVEY_STATICSKYSINGLE_DB = 0 $SURVEY_EXEC = 120 @@ -78,4 +82,10 @@ active true end + task survey.lapgroup + active true + end + task survey.relexp + active true + end end @@ -118,4 +128,10 @@ end task survey.skycal + active false + end + task survey.lapgroup + active false + end + task survey.relexp active false end @@ -494,8 +510,62 @@ macro survey.show.skycal if ($0 != 1) - echo "USAGE: survey.show.skyacl" + echo "USAGE: survey.show.skycal" break end book listbook SURVEY_SKYCAL +end + +macro survey.add.lapgroup + if ($0 != 3) + echo "USAGE: survey.add.lapgroup (label) (seq_id)" + break + end + book newpage SURVEY_LAPGROUP $1 + book setword SURVEY_LAPGROUP $1 LABEL $1 + book setword SURVEY_LAPGROUP $1 SEQ_ID $2 + book setword SURVEY_LAPGROUP $1 STATE PENDING +end + +macro survey.del.lapgroup + if ($0 != 2) + echo "USAGE: survey.del.lapgroup (label)" + break + end + book delpage SURVEY_LAPGROUP $1 +end + +macro survey.show.lapgroup + if ($0 != 1) + echo "USAGE: survey.show.lapgroup" + break + end + book listbook SURVEY_LAPGROUP +end + +macro survey.add.relexp + if ($0 != 3) + echo "USAGE: survey.add.relexp (label) (releasename)" + break + end + book newpage SURVEY_RELEXP $1 + book setword SURVEY_RELEXP $1 LABEL $1 + book setword SURVEY_RELEXP $1 RELEASE_NAME $2 + book setword SURVEY_RELEXP $1 STATE PENDING +end + +macro survey.del.relexp + if ($0 != 2) + echo "USAGE: survey.del.relexp (label)" + break + end + book delpage SURVEY_RELEXP $1 +end + +macro survey.show.relexp + if ($0 != 1) + echo "USAGE: survey.show.relexp" + break + end + book listbook SURVEY_RELEXP end @@ -1561,2 +1631,141 @@ end end + +task survey.lapgroup + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.lapgroup.log + stderr $LOGDIR/survey.lapgroup.log + + task.exec + book npages SURVEY_LAPGROUP -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_LAPGROUP 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_LAPGROUP $i -var label + book setword SURVEY_LAPGROUP $label STATE NEW + end + book getpage SURVEY_LAPGROUP 0 -var label -key STATE NEW + + # Select different database + $SURVEY_LAPGROUP_DB ++ + if ($SURVEY_LAPGROUP_DB >= $DB:n) set SURVEY_LAPGROUP_DB = 0 + end + + book setword SURVEY_LAPGROUP $label STATE DONE + book getword SURVEY_LAPGROUP $label SEQ_ID -var SEQ_ID + + # For now the list of filters is hardcoded here + $run = laptool -definegroup -seq_id $SEQ_ID -set_label $label -filter g.00000 -filter r.00000 -filter i.00000 -filter z.00000 -filter y.00000 + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_LAPGROUP_DB + option $DB:$SURVEY_LAPGROUP_DB + end + + # echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.relexp + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.relexp.log + stderr $LOGDIR/survey.relexp.log + + task.exec + book npages SURVEY_RELEXP -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_RELEXP 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_RELEXP $i -var label + book setword SURVEY_RELEXP $label STATE NEW + end + book getpage SURVEY_RELEXP 0 -var label -key STATE NEW + + # Select different database + $SURVEY_RELEXP_DB ++ + if ($SURVEY_RELEXP_DB >= $DB:n) set SURVEY_RELEXP_DB = 0 + end + + book setword SURVEY_RELEXP $label STATE DONE + book getword SURVEY_RELEXP $label RELEASE_NAME -var RELEASE_NAME + + $run = releasetool -definerelexp -label $label -release_name $RELEASE_NAME -set_state processed + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_RELEXP_DB + option $DB:$SURVEY_RELEXP_DB + end + + # echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /tags/ipp-20130307/ippTools/share/Makefile.am =================================================================== --- /tags/ipp-20130307/ippTools/share/Makefile.am (revision 35488) +++ /tags/ipp-20130307/ippTools/share/Makefile.am (revision 35489) @@ -457,4 +457,7 @@ laptool_WSdiff_check.sql \ laptool_stacks.sql \ + laptool_pendinggroup.sql \ + laptool_revertgroup.sql \ + laptool_filtersforgroup.sql \ vptool_find_rawexp.sql \ vptool_pendingimfile.sql \ @@ -470,4 +473,5 @@ releasetool_definerelstack.sql \ releasetool_definerelstack_with_skycal.sql \ + releasetool_listrelstack.sql \ releasetool_definerelgroup_select_lap.sql \ releasetool_definerelgroup_select_data_group.sql \ Index: /tags/ipp-20130307/ippTools/share/addtool_find_processedexp_cam.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/addtool_find_processedexp_cam.sql (revision 35488) +++ /tags/ipp-20130307/ippTools/share/addtool_find_processedexp_cam.sql (revision 35489) @@ -1,5 +1,11 @@ SELECT addProcessedExp.*, - addRun.workdir + addRun.label, + addRun.workdir, + camRun.cam_id, + camRun.label as cam_label, + camRun.data_group as cam_data_group, + rawExp.exp_id, + exp_name FROM addProcessedExp JOIN addRun Index: /tags/ipp-20130307/ippTools/share/laptool_filtersforgroup.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/laptool_filtersforgroup.sql (revision 35489) +++ /tags/ipp-20130307/ippTools/share/laptool_filtersforgroup.sql (revision 35489) @@ -0,0 +1,2 @@ +SELECT DISTINCT filter +FROM lapGroup JOIN lapRun USING(seq_id, tess_id, projection_cell) Index: /tags/ipp-20130307/ippTools/share/laptool_pendinggroup.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/laptool_pendinggroup.sql (revision 35489) +++ /tags/ipp-20130307/ippTools/share/laptool_pendinggroup.sql (revision 35489) @@ -0,0 +1,4 @@ +SELECT DISTINCT lapGroup.*, + lapRun.dist_group +FROM lapGroup JOIN lapRun USING(seq_id, tess_id, projection_cell) +WHERE lapGroup.state ='new' AND lapGroup.fault = 0 Index: /tags/ipp-20130307/ippTools/share/laptool_revertgroup.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/laptool_revertgroup.sql (revision 35489) +++ /tags/ipp-20130307/ippTools/share/laptool_revertgroup.sql (revision 35489) @@ -0,0 +1,3 @@ +UPDATE lapGroup +SET fault = 0 +WHERE fault != 0 Index: /tags/ipp-20130307/ippTools/share/pstamptool_completedreq.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/pstamptool_completedreq.sql (revision 35488) +++ /tags/ipp-20130307/ippTools/share/pstamptool_completedreq.sql (revision 35489) @@ -8,4 +8,5 @@ SELECT count(*) FROM pstampJob WHERE pstampJob.req_id = pstampRequest.req_id - AND pstampJob.state != 'stop' + AND pstampJob.state != 'stop' + AND pstampJob.state != 'cancel' ) = 0 Index: /tags/ipp-20130307/ippTools/share/pstamptool_getdependent.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/pstamptool_getdependent.sql (revision 35488) +++ /tags/ipp-20130307/ippTools/share/pstamptool_getdependent.sql (revision 35489) @@ -1,3 +1,3 @@ SELECT DISTINCT pstampDependent.* FROM pstampDependent -WHERE pstampDependent.state = 'new' +WHERE (pstampDependent.state = 'new' OR pstampDependent.state = 'hold') Index: /tags/ipp-20130307/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/pxadmin_create_tables.sql (revision 35488) +++ /tags/ipp-20130307/ippTools/share/pxadmin_create_tables.sql (revision 35489) @@ -1,3 +1,3 @@ -CREATE TABLE dbversion ( +eREATE TABLE dbversion ( schema_version VARCHAR(64), updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP @@ -2172,4 +2172,5 @@ priority INT, PRIMARY KEY(rel_id), + UNIQUE KEY(surveyID, release_name), KEY(release_name), KEY(release_state), @@ -2196,4 +2197,5 @@ PRIMARY KEY (relexp_id), FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), + UNIQUE KEY(rel_id, exp_id), KEY (state), KEY (fault), @@ -2224,6 +2226,6 @@ time_stamp DATETIME, PRIMARY KEY (relstack_id), + UNIQUE KEY (rel_id, tess_id, skycell_id, filter, stack_type, mjd_obs), KEY (tess_id, skycell_id), - KEY (rel_id, tess_id, skycell_id, filter, mjd_obs), KEY (stack_type), KEY (mjd_obs), @@ -2258,8 +2260,11 @@ CREATE TABLE lapGroup ( seq_id BIGINT, + tess_id VARCHAR(64), projection_cell VARCHAR(64), state VARCHAR(16), + label VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time group was registered fault SMALLINT, - PRIMARY KEY(seq_id, projection_cell), + PRIMARY KEY(seq_id, tess_id, projection_cell), KEY(state), KEY(fault), Index: /tags/ipp-20130307/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql (revision 35488) +++ /tags/ipp-20130307/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql (revision 35489) @@ -1,4 +1,6 @@ SELECT relExp.relexp_id -FROM relGroup JOIN lapRun USING(lap_id) +FROM relGroup + JOIN lapRun USING(lap_id) + JOIN lapExp USING(lap_id) JOIN relExp using(rel_id, exp_id) WHERE relExp.group_id = 0 Index: /tags/ipp-20130307/ippTools/share/releasetool_definerelstack.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/releasetool_definerelstack.sql (revision 35488) +++ /tags/ipp-20130307/ippTools/share/releasetool_definerelstack.sql (revision 35489) @@ -6,5 +6,5 @@ stackRun.tess_id, stackRun.filter, - 0 AS mjd_obs, + stackSumSkyfile.mjd_obs, 0 AS zpt_obs, 0 AS zpt_stdev @@ -12,5 +12,13 @@ FROM ippRelease JOIN stackRun -LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter) - +JOIN stackSumSkyfile using(stack_id) +LEFT JOIN relStack AS previousRelStack + ON previousRelStack.rel_id = ippRelease.rel_id + AND previousRelStack.tess_id = stackRun.tess_id + AND previousRelStack.skycell_id = stackRun.skycell_id + AND previousRelStack.filter = stackRun.filter + -- JOIN hook %s WHERE previousRelStack.relstack_id IS NULL + AND stackRun.state ='full' + AND stackSumSkyfile.quality = 0 + AND stackSumSkyfile.fault = 0 Index: /tags/ipp-20130307/ippTools/share/releasetool_definerelstack_with_skycal.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/releasetool_definerelstack_with_skycal.sql (revision 35488) +++ /tags/ipp-20130307/ippTools/share/releasetool_definerelstack_with_skycal.sql (revision 35489) @@ -13,7 +13,14 @@ JOIN stackRun JOIN stackSumSkyfile USING(stack_id) -LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter) JOIN skycalRun ON skycalRun.stack_id = stackRun.stack_id JOIN skycalResult ON skycalRun.skycal_id = skycalResult.skycal_id +-- LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter) +LEFT JOIN relStack AS previousRelStack + ON ippRelease.rel_id = previousRelStack.rel_id + AND stackRun.tess_id = previousRelStack.tess_id + AND stackRun.skycell_id = previousRelStack.skycell_id + AND stackRun.filter = previousRelStack.filter +-- JOIN Hook %s WHERE previousRelStack.relstack_id IS NULL + AND skycalRun.state = 'full' Index: /tags/ipp-20130307/ippTools/share/releasetool_listrelexp.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/releasetool_listrelexp.sql (revision 35488) +++ /tags/ipp-20130307/ippTools/share/releasetool_listrelexp.sql (revision 35489) @@ -25,4 +25,5 @@ warpRun.warp_id, warpRun.state as warp_state, + warpRun.tess_id, rawExp.filter, rawExp.dateobs, Index: /tags/ipp-20130307/ippTools/share/releasetool_listrelstack.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/releasetool_listrelstack.sql (revision 35489) +++ /tags/ipp-20130307/ippTools/share/releasetool_listrelstack.sql (revision 35489) @@ -0,0 +1,19 @@ +SELECT + release_name, + surveyName, + relStack.*, + stackSumSkyfile.path_base AS stack_path_base, + stackSumSkyfile.quality, + stackRun.state AS stack_state, + stackRun.data_group AS stack_data_group, + skycalResult.fwhm_major, + skycalResult.path_base AS skycal_path_base, + skycalRun.data_group AS skycal_data_group, + 'GPC1' AS camera +FROM relStack +JOIN ippRelease USING(rel_id) +JOIN survey USING(surveyID) +JOIN stackRun USING(stack_id, filter, tess_id, skycell_id) +JOIN stackSumSkyfile USING(stack_id) +LEFT JOIN skycalRun USING(skycal_id) +LEFT JOIN skycalResult USING(skycal_id) Index: /tags/ipp-20130307/ippTools/share/releasetool_pendingrelgroup.sql =================================================================== --- /tags/ipp-20130307/ippTools/share/releasetool_pendingrelgroup.sql (revision 35488) +++ /tags/ipp-20130307/ippTools/share/releasetool_pendingrelgroup.sql (revision 35489) @@ -1,4 +1,7 @@ -SELECT relGroup.* +SELECT + relGroup.*, + ippRelease.release_name FROM relGroup JOIN ippRelease USING(rel_id) WHERE relGroup.state = 'new' + AND relGroup.fault = 0 Index: /tags/ipp-20130307/ippTools/src/laptool.c =================================================================== --- /tags/ipp-20130307/ippTools/src/laptool.c (revision 35488) +++ /tags/ipp-20130307/ippTools/src/laptool.c (revision 35489) @@ -34,4 +34,12 @@ static bool inactiveexpMode(pxConfig *config); +// Groups +static bool definegroupMode(pxConfig *config); +static bool pendinggroupMode(pxConfig *config); +static bool filtersforgroupMode(pxConfig *config); +static bool updategroupMode(pxConfig *config); +static bool revertgroupMode(pxConfig *config); +static bool listgroupMode(pxConfig *config); + # define MODECASE(caseName, func) \ case caseName: \ @@ -67,4 +75,12 @@ MODECASE(LAPTOOL_MODE_INACTIVEEXP, inactiveexpMode); + + MODECASE(LAPTOOL_MODE_DEFINEGROUP, definegroupMode); + MODECASE(LAPTOOL_MODE_PENDINGGROUP, pendinggroupMode); + MODECASE(LAPTOOL_MODE_FILTERSFORGROUP, filtersforgroupMode); + MODECASE(LAPTOOL_MODE_UPDATEGROUP, updategroupMode); + MODECASE(LAPTOOL_MODE_REVERTGROUP, revertgroupMode); + MODECASE(LAPTOOL_MODE_LISTGROUP, listgroupMode); + default: psAbort("invalid option (this should not happen)"); @@ -900,3 +916,443 @@ return(true); } - +// --------------------------- +// Group level (a collection of completed lapRuns for a given lapSequence projection cell and collection of filters + +static bool definegroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(seq_id, config->args, "-seq_id", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // the following insures that the -filter argument has been set up properly (adapted from pxAddLabelSearchArgs) + psMetadataItem *item = psMetadataLookup(config->args, "-filter"); + psAssert (item, "-filter argument not found in config->args"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "%s should be a multi container", "filter"); + psAssert (item->data.list->n, "%s should at least have a place-holder", "filter"); + psMetadataItem *entry = (psMetadataItem *)item->data.list->head->data; + psAssert (entry, "%s should at least have a place-holder", "filter"); + // end of checking + + // Now if the ony entry is the place-holder then the user supplied no -filter arguments + // which are required + if (!entry->data.str) { + psError(PXTOOLS_ERR_ARGUMENTS, true, "at least one -filter is required"); + return false; + } + + int nFilters = item->data.list->n; + + psString query = psStringCopy("SELECT seq_id, tess_id, projection_cell,\n"); + + // We construct a query joining completed lap runs with the same lapSequence.seq_id, + // tess_id, and projection_cell + + // select the lap_ids for the various filters. These are only used for debugging. + psStringAppend(&query, "lap_id_%d", 0); + for (int i = 1; i < nFilters; i++) { + psStringAppend(&query, ", lap_id_%d", i); + } + + + // sub query for each supplied filter + char * lapRunForFilter = "(\nSELECT seq_id, tess_id, projection_cell, lap_id as 'lap_id_%d'\n" + "FROM lapRun\n" + "WHERE filter LIKE '%s'\n" + " AND lapRun.seq_id = %"PRId64"\n" + " AND (lapRun.state = 'done' or lapRun.state = 'full')\n" + " ) as lap_%d\n"; + + // loop over supplied filters and flesh out the query using the format above + psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true); + psMetadataItem *filterItem = NULL; + int i = -1; + while ((filterItem = psListGetAndIncrement(iter))) { + ++i; + if (i == 0) { + psStringAppend(&query, "\nFROM\n"); + } else { + psStringAppend(&query, "\nJOIN\n"); + } + + psString filter = filterItem->data.str; + psStringAppend(&query, lapRunForFilter, i, filter, seq_id, i); + + if (i != 0) { + psStringAppend(&query, "USING (seq_id, tess_id, projection_cell)\n"); + } + } + // now join to lapGroup + psStringAppend(&query, "\nLEFT JOIN lapGroup USING(seq_id, tess_id, projection_cell)\n"); + + // we only want projection cells which do not already of an entry + psStringAppend(&query, "\nWHERE lapGroup.projection_cell IS NULL\n"); + psStringAppend(&query, "\nORDER by projection_cell\n"); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + if (!ippdbPrintMetadatas(stdout, output, "new_lapGroups", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + psString last_projection_cell = NULL; + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psString tess_id = psMetadataLookupStr(NULL, row, "tess_id"); + psString projection_cell = psMetadataLookupStr(NULL, row, "projection_cell"); + if (last_projection_cell && !strcmp(last_projection_cell, projection_cell)) { + // duplicate lap runs for a filter will generate multiple rows + // Since we care about projection_cells not lapRuns per se this is not problem. + // Skip any duplicates. + continue; + } + last_projection_cell = projection_cell; + + lapGroupRow *group = lapGroupRowAlloc( + seq_id, + tess_id, + projection_cell, + "new", // state + label, + now, // registered + 0 // fault + ); + if (!group) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc lapGroup object"); + psFree(output); + psFree(now); + return(false); + } + + + if (!lapGroupInsertObject(config->dbh, group)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return(true); + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(now); + psFree(output); + + return(true); +} + +static bool pendinggroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "lapRun.label", "=="); + + psString query = pxDataGet("laptool_pendinggroup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapGroup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +static bool filtersforgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + + psString query = pxDataGet("laptool_filtersforgroup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "filters", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool updategroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // check that all of the keys that define a lapGroup are supplied + PXOPT_LOOKUP_S64(seq_id, config->args, "-seq_id", true, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + + psMetadata *values = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, values, "-set_state", "state", "=="); + PXOPT_COPY_STR(config->args, values, "-set_label", "label", "=="); + PXOPT_COPY_S16(config->args, values, "-set_fault", "fault", "=="); + + if (!psListLength(values->list)) { + psFree(values); + psFree(where); + psError(PXTOOLS_ERR_ARGUMENTS, true, "must set at least one column"); + return false; + } + + long rows = psDBUpdateRows(config->dbh, "lapGroup", where, values); + if (rows < 1) { + psFree(values); + psError(PXTOOLS_ERR_SYS, true, "failed to update lapGroup for %" PRId64 " %s %s", seq_id, tess_id, projection_cell); + return false; + } + psFree(values); + + return(true); +} + +static bool revertgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search arguments are required"); + return false; + } + + psString query = pxDataGet("laptool_revertgroup.sql"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("laptool", PS_LOG_INFO, "Updated %" PRIu64 " lapGroups", affected); + + return true; +} + +static bool listgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psError(PXTOOLS_ERR_SYS, true, "not yet implemented"); + return false; + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-seq_name", "name", "LIKE"); + + psString query = pxDataGet("laptool_listsequence.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapSequence", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /tags/ipp-20130307/ippTools/src/laptool.h =================================================================== --- /tags/ipp-20130307/ippTools/src/laptool.h (revision 35488) +++ /tags/ipp-20130307/ippTools/src/laptool.h (revision 35489) @@ -20,5 +20,11 @@ LAPTOOL_MODE_UPDATEEXP, LAPTOOL_MODE_DIFFCHECK, - LAPTOOL_MODE_INACTIVEEXP + LAPTOOL_MODE_INACTIVEEXP, + LAPTOOL_MODE_DEFINEGROUP, + LAPTOOL_MODE_PENDINGGROUP, + LAPTOOL_MODE_FILTERSFORGROUP, + LAPTOOL_MODE_UPDATEGROUP, + LAPTOOL_MODE_REVERTGROUP, + LAPTOOL_MODE_LISTGROUP, } laptoolMode; Index: /tags/ipp-20130307/ippTools/src/laptoolConfig.c =================================================================== --- /tags/ipp-20130307/ippTools/src/laptoolConfig.c (revision 35488) +++ /tags/ipp-20130307/ippTools/src/laptoolConfig.c (revision 35489) @@ -97,5 +97,4 @@ ADD_OPT(S64, updaterunArgs, "-set_quick_sass_id", "set quick stack sass_id", 0); ADD_OPT(S64, updaterunArgs, "-set_final_sass_id", "set final stack sass_id", 0); - // -pendingexp @@ -150,4 +149,52 @@ ADD_OPT(U64, inactiveexpArgs, "-limit", "limit result set to N items", 0); + // -definegroup + psMetadata *definegroupArgs = psMetadataAlloc(); + ADD_OPT(S64, definegroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + ADD_OPT(Str, definegroupArgs, "-set_label", "set label (required)", 0); + + psMetadataAddStr(definegroupArgs, PS_LIST_TAIL, + "-filter", PS_META_DUPLICATE_OK, + "search by filter (LIKE comparison, multiple OK)", NULL); + ADD_OPT(Bool,definegroupArgs, "-pretend", "do not actuallym modify the database", false); + ADD_OPT(Bool,definegroupArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, definegroupArgs, "-limit", "limit result set to N items", 0); + + // -pendinggroup + psMetadata *pendinggroupArgs = psMetadataAlloc(); + psMetadataAddStr(pendinggroupArgs, PS_LIST_TAIL, + "-label", PS_META_DUPLICATE_OK, + "search by label (multiple OK)", NULL); + ADD_OPT(S64, pendinggroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + + ADD_OPT(Bool,pendinggroupArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, pendinggroupArgs, "-limit", "limit result set to N items", 0); + + // -updategroup + psMetadata *updategroupArgs = psMetadataAlloc(); + ADD_OPT(S64, updategroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + ADD_OPT(Str, updategroupArgs, "-tess_id", "search by tess_id (required)", 0); + ADD_OPT(Str, updategroupArgs, "-projection_cell", "search by projection_cell (required)", 0); + ADD_OPT(Str, updategroupArgs, "-set_state", "set state", 0); + ADD_OPT(S16, updategroupArgs, "-set_fault", "set fault code", INT16_MAX); + ADD_OPT(Str, updategroupArgs, "-set_label", "set label", 0); + + // -revertgroup + psMetadata *revertgroupArgs = psMetadataAlloc(); + psMetadataAddStr(revertgroupArgs, PS_LIST_TAIL, + "-label", PS_META_DUPLICATE_OK, + "search by label (multiple OK)", NULL); + ADD_OPT(S64, revertgroupArgs, "-seq_id", "search by lap sequence ID", 0); + ADD_OPT(Str, revertgroupArgs, "-tess_id", "search by tess_id", 0); + ADD_OPT(Str, revertgroupArgs, "-projection_cell", "search by projection_cell", 0); + ADD_OPT(S16, revertgroupArgs, "-fault", "fault code", INT16_MAX); + + // -filtersforgroup + psMetadata *filtersforgroupArgs = psMetadataAlloc(); + ADD_OPT(S64, filtersforgroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + ADD_OPT(Str, filtersforgroupArgs, "-tess_id", "search by tess_id (required)", 0); + ADD_OPT(Str, filtersforgroupArgs, "-projection_cell", "search by projection_cell (required)", 0); + ADD_OPT(Bool,filtersforgroupArgs, "-simple", "use the simple output format", false); + psMetadata *argSets = psMetadataAlloc(); @@ -166,4 +213,9 @@ PXOPT_ADD_MODE("-diffcheck", "", LAPTOOL_MODE_DIFFCHECK, diffcheckArgs); PXOPT_ADD_MODE("-inactiveexp", "", LAPTOOL_MODE_INACTIVEEXP, inactiveexpArgs); + PXOPT_ADD_MODE("-definegroup", "", LAPTOOL_MODE_DEFINEGROUP, definegroupArgs); + PXOPT_ADD_MODE("-pendinggroup", "", LAPTOOL_MODE_PENDINGGROUP, pendinggroupArgs); + PXOPT_ADD_MODE("-filtersforgroup", "", LAPTOOL_MODE_FILTERSFORGROUP, filtersforgroupArgs); + PXOPT_ADD_MODE("-updategroup", "", LAPTOOL_MODE_UPDATEGROUP, updategroupArgs); + PXOPT_ADD_MODE("-revertgroup", "", LAPTOOL_MODE_REVERTGROUP, revertgroupArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /tags/ipp-20130307/ippTools/src/pstamptool.c =================================================================== --- /tags/ipp-20130307/ippTools/src/pstamptool.c (revision 35488) +++ /tags/ipp-20130307/ippTools/src/pstamptool.c (revision 35489) @@ -49,7 +49,4 @@ static bool revertjobMode(pxConfig *config); -# if (1) -// these are unused functions -- since they are 'static', this raises an warning -// XXX: They actually are used static bool addprojectMode(pxConfig *config); static bool projectMode(pxConfig *config); @@ -63,5 +60,4 @@ static bool listfileMode(pxConfig *config); static bool deletefileMode(pxConfig *config); -# endif # define MODECASE(caseName, func) \ @@ -103,5 +99,4 @@ MODECASE(PSTAMPTOOL_MODE_STOPDEPENDENTJOB, stopdependentjobMode); MODECASE(PSTAMPTOOL_MODE_REVERTJOB, revertjobMode); -# if (1) MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode); MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode); @@ -115,5 +110,5 @@ MODECASE(PSTAMPTOOL_MODE_LISTFILE, listfileMode); MODECASE(PSTAMPTOOL_MODE_DELETEFILE, deletefileMode); -# endif + default: psAbort("invalid option (this should not happen)"); @@ -452,5 +447,7 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); - PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-username", "username", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -458,5 +455,5 @@ if (!psListLength(where->list)) { - psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied"); + psError(PS_ERR_UNKNOWN, true, "search paramters are required"); return false; } @@ -490,8 +487,8 @@ if (!psArrayLength(output)) { psTrace("pstamptool", PS_LOG_INFO, "request not found"); - // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for + // This causes main to exit with PS_EXIT_DATA_ERROR which the pstamp scripts are looking for psError(PXTOOLS_ERR_CONFIG, true, "request not found"); psFree(output); - // we return false so that the caller can determine that a request does not exist + // we return false so that the caller can easily determine that a request does not exist return false; } @@ -593,6 +590,7 @@ PXOPT_COPY_S32(config->args, where, "-fault", "fault", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); - PXOPT_COPY_STR(config->args, where, "-reqType", "reqType", "=="); - PXOPT_COPY_STR(config->args, where, "-name", "name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-reqType", "reqType", "=="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-username", "username", "=="); PXOPT_COPY_TIME(config->args, where, "-timestamp_begin", "timestamp", ">="); PXOPT_COPY_TIME(config->args, where, "-timestamp_end", "timestamp", "<="); @@ -621,4 +619,7 @@ if (outdir) { psStringAppend(&query, ", outdir = '%s'", outdir); + } + if (username) { + psStringAppend(&query, ", username = '%s'", username); } if (clearfault) { @@ -1233,4 +1234,5 @@ PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false); PXOPT_LOOKUP_BOOL(no_create, config->args, "-no_create", false); + PXOPT_LOOKUP_BOOL(hold, config->args, "-hold", false); psMetadata *where = psMetadataAlloc(); @@ -1240,5 +1242,5 @@ PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); - // start a transaction eraly so it will contain any row level locks + // start a transaction early so it will contain any row level locks if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1264,4 +1266,7 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1271,4 +1276,7 @@ if (!output) { psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1279,4 +1287,7 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1284,10 +1295,51 @@ if (fault > 0) { fprintf(stderr, "existing dependent has fault %d\n", fault); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + // return the fault to the client. This is used by the postage stamp parser exit (fault); } + bool commit = false; + // Check the state of the exisiting dependent. By query it is either + // new or hold + psString state = psMetadataLookupStr(NULL, dep, "state"); + if (!hold && !strcmp(state, "hold")) { + // There is a dependent for this component but it's state is hold. + // This client needs one that will run. + // Update the state + psString updateQuery = NULL; + psStringAppend(&updateQuery, "UPDATE pstampDependent SET state = 'new' " + "\nWHERE stage = '%s' AND imagedb = '%s' AND stage_id = %"PRId64 " AND component = '%s'", + stage, imagedb, stage_id, component); + + if (!p_psDBRunQuery(config->dbh, updateQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(updateQuery); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + // set flag to commit this change + commit = true; + psFree(updateQuery); + } + // print the dep_id for the user printf("%" PRId64 "\n", dep_id); psFree(output); - if (!psDBRollback(config->dbh)) { + // now either commit the change or rollback the transaction which releases the lock + if (commit) { + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } else if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; } return true; @@ -1307,5 +1359,5 @@ config->dbh, 0, // dep_id - "new", // state + hold ? "hold" : "new", // state stage, stage_id, Index: /tags/ipp-20130307/ippTools/src/pstamptoolConfig.c =================================================================== --- /tags/ipp-20130307/ippTools/src/pstamptoolConfig.c (revision 35488) +++ /tags/ipp-20130307/ippTools/src/pstamptoolConfig.c (revision 35489) @@ -90,4 +90,6 @@ psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "list by req_id", 0); psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0, "list by name", NULL); + psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-username", 0, "list by user name", NULL); + psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-state", 0, "list by state", NULL); psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0, "req_id to not list", 0); psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -107,5 +109,6 @@ psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "search by reqType", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "search by reqType (LIKE comparsion)", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "search by reqType (LIKE comparsion)", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-username", 0, "search by username", NULL); psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddTime(updatereqArgs, PS_LIST_TAIL, "-timestamp_begin", 0, "search by timestamp (>=)", NULL); @@ -220,4 +223,5 @@ psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-need_magic", 0, "define need_magic", false); psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory for dependent (required)", NULL); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-hold", 0, "if creating new dependent set it's state to hold", false); psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no matching dependent do not create one", false); Index: /tags/ipp-20130307/ippTools/src/releasetool.c =================================================================== --- /tags/ipp-20130307/ippTools/src/releasetool.c (revision 35488) +++ /tags/ipp-20130307/ippTools/src/releasetool.c (revision 35489) @@ -702,4 +702,8 @@ PXOPT_COPY_STR(config->args, where, "-cam_data_group", "camRun.data_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); PXOPT_COPY_STR(config->args, where, "-surveyName", "survey.surveyName", "LIKE"); @@ -715,4 +719,8 @@ psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; + } + + if (skycell_id) { + psStringAppend(&query, "\nJOIN warpSkyfile ON warpRun.warp_id = warpSkyfile.warp_id AND warpRun.tess_id = warpSkyfile.tess_id"); } @@ -741,5 +749,5 @@ if (priority_order) { - psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority"); + psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority DESC"); } @@ -787,7 +795,7 @@ bool includeSkycal = false; + PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE"); PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "=="); if (psListLength(where->list)) { includeSkycal = true; @@ -795,5 +803,5 @@ PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE"); PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); @@ -832,4 +840,5 @@ PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); // find the parameters of all the exposures that we want to add to the release @@ -847,16 +856,41 @@ } + psString joinConditions = psStringCopy(""); + // Finish up the selectors for the JOIN to previousRelStack + // join to rows with our stack_type + psStringAppend(&joinConditions, "\nAND previousRelStack.stack_type = '%s'", stack_type); + + if (!strcmp(stack_type, "nightly")) { + // detect nightly stack entries that already exist for this day + psStringAppend(&joinConditions, "\nAND previousRelStack.mjd_obs = floor(stackSumSkyfile.mjd_obs)"); + } + + // Add in the where conditions psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query, "\nAND %s", whereClause); psFree(whereClause); + if (!strcmp(stack_type, "nightly")) { + // avoid stacks with NAN mjd_obs. These are old skycells that were lost due to system failure + // prior to the time that the mjd_obs was extracted from the headers and added to stackSumSkyfile + psStringAppend(&query, "\nAND stackSumSkyfile.mjd_obs IS NOT NULL"); + } + psFree(where); - if (!p_psDBRunQuery(config->dbh, query)) { + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, joinConditions)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); + psFree(joinConditions); return false; } psFree(query); + psFree(joinConditions); psArray *output = p_psDBFetchResult(config->dbh); @@ -968,23 +1002,26 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-relstack_id", "relStack.relstack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "relStack.stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "relStack.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-relstack_id", "relStack.relstack_id", "=="); PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE"); pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "=="); - PXOPT_COPY_STR(config->args, where, "-state", "relExp.state", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin","rawExp.dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); - PXOPT_COPY_F32(config->args, where, "-fwhm_min", "skycalResult.fwhm_major", ">="); - PXOPT_COPY_F32(config->args, where, "-fwhm_max", "skycalresult.fwhm_major", "<="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_ame", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "=="); - PXOPT_COPY_S64(config->args, where, "-chip_id", "relExp.chip_id", "=="); - PXOPT_COPY_S64(config->args, where, "-cam_id", "relExp.cam_id", "=="); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-chip_data_group", "chipRun.data_group", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-cam_data_group", "camRun.data_group", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "relStack.state", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "relStack.filter", "LIKE"); + PXOPT_COPY_F32(config->args, where, "-mjd_min", "stackSumSkyfile.mjd_obs", ">="); + PXOPT_COPY_F32(config->args, where, "-mjd_max", "stackSumSkyfile.mjd_obs", "<="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "relStack.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "relStack.skycell_id", "LIKE"); +// PXOPT_COPY_STR(config->args, where, "-stack_type", "relStack.stack_type", "=="); + PXOPT_COPY_STR(config->args, where, "-stack_data_group", "stackRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-surveyName", "survey.surveyName", "LIKE"); PXOPT_COPY_S32(config->args, where, "-rel_id", "relExp.rel_id", "=="); + pxskycellAddWhere(config, where); + + PXOPT_COPY_F32(config->args, where, "-fwhm_min", "IFNULL(skycalResult.fwhm_major, 999)", ">="); + PXOPT_COPY_F32(config->args, where, "-fwhm_max", "IFNULL(skycalResult.fwhm_major, 0)", "<="); PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false); @@ -993,13 +1030,9 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = pxDataGet("releasetool_listrelexp.sql"); + pxAddLabelSearchArgs (config, where, "-stack_type", "relStack.stack_type", "=="); + + psString query = pxDataGet("releasetool_listrelstack.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - psString where2 = NULL; - if (!pxspaceAddWhere(config, &where2, "rawExp")) { - psError(psErrorCodeLast(), false, "pxspaceAddWhere failed"); return false; } @@ -1009,6 +1042,4 @@ psStringAppend(&query, "\nWHERE %s", whereClause); psFree(whereClause); - } else if (where2) { - psStringAppend(&query, "\nWHERE "); } else { psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); @@ -1017,11 +1048,6 @@ } - if (where2) { - psStringAppend(&query, "\n%s", where2); - psFree(where2); - } - if (priority_order) { - psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority"); + psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority DESC"); } @@ -1051,5 +1077,5 @@ } - if (!ippdbPrintMetadatas(stdout, output, "relExp", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "relStack", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -1270,4 +1296,7 @@ } if (!psArrayLength(exposures)) { + fprintf(stderr, "no exposures found for lap_id %" PRId64 "\n", lap_id); +#ifdef notdef + continue; psFree(now); psFree(output); @@ -1278,4 +1307,5 @@ } return false; +#endif } @@ -1301,5 +1331,5 @@ psFree(exposures); - // set state of relGroup to 'new' + // update the state of relGroup to 'new' if (!p_psDBRunQueryF(config->dbh, "UPDATE relGroup set state = 'new' WHERE group_id = %d", group_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1325,6 +1355,84 @@ static bool updaterelgroupMode(pxConfig *config) { - psError(PS_ERR_UNKNOWN, true, "not yet implemented"); - return false; + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S32(group_id, config->args, "-group_id", false, false); + PXOPT_LOOKUP_STR(group_name, config->args, "-group_name", false, false); + if (!group_id && !group_name) { + psError(PXTOOLS_ERR_CONFIG, true, "either group_id or group_name is required\n"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(exp_list_path, config->args, "-set_exp_list_path", false, false); + PXOPT_LOOKUP_STR(new_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); + + if (!state && !exp_list_path && !new_label && !fault && !clearfault) { + psError(PXTOOLS_ERR_CONFIG, true, "must set at least one column\n"); + return false; + } + + if (fault && clearfault) { + psError(PXTOOLS_ERR_CONFIG, true, "cannot set and clear fault at same time\n"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-group_id", "relGroup.group_id", "=="); + PXOPT_COPY_STR(config->args, where, "-group_name", "relGroup.group_name", "=="); + // XXX TODO if label is enabled (for changing label or state for a block) we should + // disallow setting some parameters such as exp_list_path + // PXOPT_COPY_S64(config->args, where, "-label", "relGroup.label", "=="); + // make sure that we have enough parameters to identify the relGroup to change + if (!psListLength(where->list)){ + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); + psFree(where); + return false; + } + + psString sep = ""; + psString comma = ", "; + psString query = psStringCopy("UPDATE relGroup SET"); + if (state) { + psStringAppend(&query, "%s relGroup.state = '%s'", sep, state); + sep = comma; + } + if (exp_list_path) { + psStringAppend(&query, "%s relGroup.exp_list_path = '%s'", sep, exp_list_path); + sep = comma; + } + if (new_label) { + psStringAppend(&query, "%s relGroup.label = '%s'", sep, new_label); + sep = comma; + } + if (fault) { + psStringAppend(&query, "%s relGroup.fault = %d", sep, fault); + sep = comma; + } + if (clearfault) { + psStringAppend(&query, "%s relGroup.fault = 0", sep); + sep = comma; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " ippReleases", affected); + + + psFree(query); + + return true; } @@ -1410,5 +1518,5 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); PXOPT_COPY_F32(config->args, where, "-fwhm_min", "skycalResult.fwhm_major", ">="); - PXOPT_COPY_F32(config->args, where, "-fwhm_max", "skycalresult.fwhm_major", "<="); + PXOPT_COPY_F32(config->args, where, "-fwhm_max", "skycalResult.fwhm_major", "<="); PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_ame", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "=="); Index: /tags/ipp-20130307/ippTools/src/releasetoolConfig.c =================================================================== --- /tags/ipp-20130307/ippTools/src/releasetoolConfig.c (revision 35488) +++ /tags/ipp-20130307/ippTools/src/releasetoolConfig.c (revision 35489) @@ -164,4 +164,6 @@ psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-cam_data_group", 0, "camRun.data_group (LIKE comparison)", NULL); psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-warp_data_group", 0, "warpRun.data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-skycell_id", 0, "select by skycell", NULL); + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-tess_id", 0, "warpRun.tess_id", NULL); psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL); @@ -195,9 +197,47 @@ psMetadataAddF32(definerelstackArgs, PS_LIST_TAIL, "-set_zpt_stdev", 0, "define zero point stdev", NAN); psMetadataAddF32(definerelstackArgs, PS_LIST_TAIL, "-set_fwhm_major", 0, "define fwhm_major", NAN); - psMetadataAddStr(definerelstackArgs, PS_LIST_TAIL, "-set_path_base", 0, "define state (required)", NULL); + psMetadataAddStr(definerelstackArgs, PS_LIST_TAIL, "-set_path_base", 0, "define state", NULL); psMetadataAddS16(definerelstackArgs, PS_LIST_TAIL, "-set_fault", 0, "define fault", 0); - psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(definerelstackArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -listrelstack + psMetadata *listrelstackArgs = psMetadataAlloc(); + psMetadataAddS64(listrelstackArgs, PS_LIST_TAIL, "-relstack_id", 0, "select by released exposure ID", 0); + psMetadataAddS64(listrelstackArgs, PS_LIST_TAIL, "-stack_id", 0, "select by stack ID", 0); + psMetadataAddS64(listrelstackArgs, PS_LIST_TAIL, "-skycal_id", 0, "select by skycal ID", 0); + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL); + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-release_state", PS_META_DUPLICATE_OK, "select by release state", NULL); + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-state", 0, "select by released stack state", NULL); + + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-filter", 0, "select by filter name (LIKE comparison)", NULL); + // psMetadataAddTime(listrelstackArgs, PS_LIST_TAIL, "-dateobs_begin", 0,"search for exposures by time (>=)", NULL); + // psMetadataAddTime(listrelstackArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + + pxskycellAddArguments(listrelstackArgs); + + psMetadataAddF32(listrelstackArgs, PS_LIST_TAIL, "-fwhm_min", 0, "search by measured seeing (>=)", NAN); + psMetadataAddF32(listrelstackArgs, PS_LIST_TAIL, "-fwhm_max", 0, "search by seeing (<=)", NAN); + + psMetadataAddF32(listrelstackArgs, PS_LIST_TAIL, "-mjd_min", 0, "search by MJD seeing (>=)", NAN); + psMetadataAddF32(listrelstackArgs, PS_LIST_TAIL, "-mjd_max", 0, "search by MJD (<=)", NAN); + + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-tess_id", 0, "select by tess_id", NULL); + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "select by skycell_id (LIKE comparision)", NULL); + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-stack_type", PS_META_DUPLICATE_OK, "select by stack_type", NULL); + + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-stack_data_group", 0, "select by stackRun.data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-skycal_data_group", 0, "select by skycalRun.data_group (LIKE comparison)", NULL); + + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL); + psMetadataAddS64(listrelstackArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); + + psMetadataAddBool(listrelstackArgs, PS_LIST_TAIL, "-priority_order", 0, "order by release priority", false); + + psMetadataAddU64(listrelstackArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrelstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -definerelgroup @@ -217,5 +257,5 @@ // parameters of the relGroup psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_type", 0, "define group_type (required)", NULL); - psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_name", 0, "define group_name (required)", NULL); + psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_name", 0, "define group_name", NULL); psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_label", 0, "define relgroup label (required)", NULL); psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -240,6 +280,6 @@ psMetadata *updaterelgroupArgs = psMetadataAlloc(); - // set the target release - psMetadataAddS32(updaterelgroupArgs, PS_LIST_TAIL, "-group_id", 0, "select by group ID (required)", 0); + psMetadataAddS32(updaterelgroupArgs, PS_LIST_TAIL, "-group_id", 0, "select by group ID", 0); + psMetadataAddStr(updaterelgroupArgs, PS_LIST_TAIL, "-group_name", 0, "select by group name", 0); // parameters of the relGroup @@ -273,4 +313,5 @@ PXOPT_ADD_MODE("-definerelstack", "define a released stack", RELEASETOOL_MODE_DEFINERELSTACK, definerelstackArgs); + PXOPT_ADD_MODE("-listrelstack", "list released stacks", RELEASETOOL_MODE_LISTRELSTACK, listrelstackArgs); PXOPT_ADD_MODE("-definerelgroup", "define a group of exposures", RELEASETOOL_MODE_DEFINERELGROUP, definerelgroupArgs); Index: /tags/ipp-20130307/pstamp/scripts/psmkreq =================================================================== --- /tags/ipp-20130307/pstamp/scripts/psmkreq (revision 35488) +++ /tags/ipp-20130307/pstamp/scripts/psmkreq (revision 35489) @@ -1,7 +1,8 @@ #!/bin/env perl ### -### pstamprequest +### psmkreq ### -### Program to make a postage stamp request table for a set of coordinates +### Program to make a postage stamp request table for a set of coordinates using supplied command +### line options and defaults ### @@ -28,6 +29,10 @@ my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base); -my ($image, $mask, $variance, $cmf, $psf, $backmdl, $inverse); -my ($unconvolved, $use_imfile_id, $no_wait); +my ($image, $mask, $variance, $jpeg, $cmf, $psf, $backmdl, $inverse); +my ($convolved, $unconvolved, $uncompressed, $use_imfile_id, $no_wait); + +# new header keywords for version 2 +my $action = 'PROCESS'; +my $email = 'null'; my $default_size = 100; @@ -52,4 +57,12 @@ my $comment; +# new request specification columns for version 2 +my $survey_name ='null'; +my $release_name = 'null'; +my $run_type ='null'; +my $fwhm_min = 0; +my $fwhm_max = 0; + + my $missing_tools; my $pstamp_request_file = can_run('pstamp_request_file') or (warn "Can't find required program pstamp_request_file" and $missing_tools = 1); @@ -62,5 +75,5 @@ GetOptions( 'list=s' => \$list, # list of coordinates if undef ra and dec or x and y are required - 'ra=s' => \$ra, # + 'ra=s' => \$ra, 'dec=s' => \$dec, 'x=s' => \$x, @@ -87,7 +100,14 @@ 'comment=s' => \$comment, + 'survey=s' => \$survey_name, + 'release=s' => \$release_name, + 'run_type=s' => \$run_type, + 'fwhm_min=s' => \$fwhm_min, + 'fwhm_max=s' => \$fwhm_max, + 'option_mask=i' => \$option_mask, 'image' => \$image, 'mask' => \$mask, + 'jpeg' => \$jpeg, 'variance' => \$variance, 'cmf' => \$cmf, @@ -96,6 +116,11 @@ 'inverse' => \$inverse, 'unconvolved' => \$unconvolved, + 'convolved' => \$convolved, + 'uncompressed' => \$uncompressed, 'use_imfile_id' => \$use_imfile_id, 'do_not_wait' => \$no_wait, + + 'action=s' => \$action, + 'email=s' => \$email, 'verbose' => \$verbose, @@ -156,4 +181,11 @@ } +if ($stage eq 'stack') { + # default is to do convolved stack + unless (defined $option_mask || $convolved) { + $unconvolved = 1; + } +} + checkFilter($filter, 'null', $filter) if $filter; checkMJD($mjd_min, 0, "") if $mjd_min; @@ -167,7 +199,9 @@ $option_mask |= $PSTAMP_SELECT_MASK if $mask; $option_mask |= $PSTAMP_SELECT_VARIANCE if $variance; + $option_mask |= $PSTAMP_SELECT_JPEG if $jpeg; + + # if no image was requested make a stamp of the image $option_mask = $PSTAMP_SELECT_IMAGE if $option_mask == 0; - # if no image was requested make a stamp of the image $option_mask |= $PSTAMP_SELECT_CMF if $cmf; @@ -207,16 +241,18 @@ } else { $rows = []; - push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max); + push @$rows, buildRow("", $comment, 1, $x, $y, $filter, $mjd_min, $mjd_max); } my ($tdf, $table_def_name) = tempfile ("/tmp/tabledef.XXXX", UNLINK => !$save_temps); -print $tdf "$req_name 1\n"; +my $line = "$req_name 2 $action $email"; +print "$line\n" if $verbose; +print $tdf "$line\n"; my $rownum = 0; foreach my $row (@$rows) { $rownum++; my $line = "$rownum $row->{ra}\t$row->{dec}\t$width $height" - . " $coord_mask $job_type $option_mask $project $req_type" + . " $coord_mask $job_type $option_mask $project $survey_name $release_name $req_type" . " $stage $id $tess_id $component $data_group" - . " $row->{filter} $row->{mjd_min} $row->{mjd_max}"; + . " $row->{filter} $row->{mjd_min} $row->{mjd_max} $run_type $fwhm_min $fwhm_max"; if ($row->{comment} and $row->{comment} ne '') { @@ -235,5 +271,5 @@ $command .= " --output $output" if $output; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + run(command => $command, verbose => $verbose); unless ($success) { print STDERR @$stderr_buf; @@ -274,9 +310,10 @@ my $row = {}; - $row->{ra} = checkRA($vals[0], $linenumber); - $row->{dec} = checkDEC($vals[1], $linenumber); - $row->{filter} = checkFilter($vals[2], $filter, $linenumber); - $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);; - $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);; + $row->{rownum} = checkRownum($vals[0], $linenumber); + $row->{ra} = checkRA($vals[1], $linenumber); + $row->{dec} = checkDEC($vals[2], $linenumber); + $row->{filter} = checkFilter($vals[3], $filter, $linenumber); + $row->{mjd_min} = checkMJD($vals[4], $mjd_min, $linenumber);; + $row->{mjd_max} = checkMJD($vals[5], $mjd_max, $linenumber);; $row->{comment} = $comment; @@ -305,5 +342,5 @@ my $linenumber = shift; - my $result; + my $result; if ($c =~ /\:/) { # sexagesmial format not valid for pixel coordinates @@ -329,4 +366,15 @@ } +sub checkRownum { + my $rownum = shift; + my $linenumber = shift; + die "rownum can not be null at $linenumber\n" unless defined $rownum; + + # XXX: For now just check that rownum is defined and a single word + # extend this. Should we require a number? + my @words = split " ", $rownum; + die "$rownum is not a vaild rownum at $linenumber\n" unless (scalar @words) == 1; +} + sub checkRA { my $ra = shift; @@ -335,4 +383,5 @@ if ($ra =~ /\:/) { + # assume RA is in hours:minutes:seconds return $checked * 360. / 24.; } else { Index: /tags/ipp-20130307/pstamp/scripts/pstamp_finish.pl =================================================================== --- /tags/ipp-20130307/pstamp/scripts/pstamp_finish.pl (revision 35488) +++ /tags/ipp-20130307/pstamp/scripts/pstamp_finish.pl (revision 35489) @@ -107,7 +107,5 @@ my ($header, $rows) = read_request_file($req_file); - my $action = $header->{ACTION}; - - if (!$header or (($action eq 'PROCESS') and !$rows)) { + if (!$header or !$rows) { # Since a request got queued, the request file must have been readable at some point my_die("failed to read request file $req_file", $req_id, $PS_EXIT_CONFIG_ERROR); @@ -128,5 +126,5 @@ my $request_fault = 0; - if ($action eq 'PROCESS') { + { # The results table definition file my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps); @@ -238,5 +236,5 @@ } else { my_die("No reglist for successful job: $job_id", $req_id, $PS_EXIT_PROG_ERROR) - if $fault eq $PSTAMP_SUCCESS; + if $job->{state} eq 'stop' and $fault eq $PSTAMP_SUCCESS; print STDERR "no reglist file for job $job_id\n" if $verbose; print $tdf "$rownum|$fault|$error_string|0|$job_id|"; @@ -273,8 +271,4 @@ } } - } elsif ($action ne 'LIST') { - my_die("Unexpected action $action found", $req_id, $PS_EXIT_PROG_ERROR); - } else { - # pstampparse did all of the work } Index: /tags/ipp-20130307/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /tags/ipp-20130307/pstamp/scripts/pstamp_job_run.pl (revision 35488) +++ /tags/ipp-20130307/pstamp/scripts/pstamp_job_run.pl (revision 35489) @@ -113,4 +113,5 @@ if ($stage eq "raw") { + # zap options that don't apply to raw stage $options &= ~($PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE); } @@ -136,4 +137,22 @@ $argString .= " -astrom $params->{astrom}"; push @file_list, $params->{astrom}; + } + + if ($options & $PSTAMP_SELECT_SOURCES) { + # Extract sources from astrometry file if provided. This will be the smf for chip stage + # or the skycal cmf for stacks + if ($params->{astrom}) { + $argString .= " -write_cmf"; + if ($stage eq 'stack') { + # Set psphot recipe to STACKPHOT so that the extended source paramters will + # be copied from the cmf file. + $argString .= " -recipe PSPHOT STACKPHOT" + } + } elsif ($params->{cmf}) { + $argString .= " -write_cmf"; + push @file_list, $params->{cmf}; + } else { + print "Could not find suitable sources file will not write cmf\n"; + } } @@ -254,7 +273,8 @@ $PSTAMP_SELECT_MASK => "mk.fits", $PSTAMP_SELECT_VARIANCE => "wt.fits", + $PSTAMP_SELECT_SOURCES => "cmf", $PSTAMP_SELECT_JPEG => "jpg"); - my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG); + my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES); foreach my $key (keys (%extensions)) { @@ -400,8 +420,8 @@ my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL); my $cmf_file; - if ($stage ne 'chip') { - # we don't ship chip stage cmf files because they may not be censored - $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF); - } +# if ($stage ne 'chip') { +# # we don't ship chip stage cmf files because they may not be censored +# $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF); +# } my $outdir = dirname($output_base); Index: /tags/ipp-20130307/pstamp/scripts/pstamp_parser_run.pl =================================================================== --- /tags/ipp-20130307/pstamp/scripts/pstamp_parser_run.pl (revision 35488) +++ /tags/ipp-20130307/pstamp/scripts/pstamp_parser_run.pl (revision 35489) @@ -158,8 +158,12 @@ my $request_fault = $PSTAMP_INVALID_REQUEST; +# default action is to process the request after parsing. This can be overridden by +# PREVIEW mode for pstamp requests +my $action = 'PROCESS'; + if (-r $uri) { # run the appropriate parse command to parse the queue the jobs for this request # first check the extension header to find the EXTNAME - $request_type = find_request_type($uri); + $request_type = find_request_type($uri, \$action); if ($request_type) { @@ -240,5 +244,13 @@ if ($success) { - $newState = 'run'; + # XXX: This bit of the postage stamp request API has slipped in here because we need to control + # the new state of the request + if ($action eq 'PROCESS') { + $newState = 'run'; + } elsif ($action eq 'PREVIEW') { + $newState = 'parsed'; + } else { + print STDERR "WARNING Ignoring unexpected value for ACTION found in request header: $action\n"; + } } else { $fault = $error_code >> 8; @@ -275,9 +287,15 @@ # find the EXTNAME in the input fits table my $file_name = shift; - my $out = `echo $file_name | fields -x 0 EXTNAME`; + my $r_action = shift; + + my $out = `echo $file_name | fields -x 0 EXTNAME ACTION`; if ($out) { # output from fields is filename value - my ($dummy, $extname) = split " ", $out; + my ($dummy, $extname, $action) = split " ", $out; + + # Set the action if it is defined in the request header + # XXX:consider doing this only if extname is PS1_PS_REQUSET + $$r_action = $action if ($action); return $extname; Index: /tags/ipp-20130307/pstamp/scripts/pstamp_request_file =================================================================== --- /tags/ipp-20130307/pstamp/scripts/pstamp_request_file (revision 35488) +++ /tags/ipp-20130307/pstamp/scripts/pstamp_request_file (revision 35489) @@ -18,4 +18,5 @@ $output, # Name of output table $req_name, + $email, $help ); @@ -25,5 +26,6 @@ 'output|o=s' => \$output, 'req_name|r=s' => \$req_name, - 'help|h' => \$help, + 'email=s' => \$email, + 'help|h' => \$help, ) or pod2usage( 2 ); @@ -54,9 +56,9 @@ value => undef }, - { name => 'USERNAME', - writetype => TSTRING, - comment => 'username for request (optional)', - value => undef - }, +# { name => 'USERNAME', +# writetype => TSTRING, +# comment => 'username for request (optional)', +# value => undef +# }, { name => 'EMAIL', writetype => TSTRING, @@ -65,4 +67,6 @@ }, ]; + +my $email_column_num = 3; # Specification of columns to write @@ -97,5 +101,8 @@ { name => 'MJD_MIN', type => 'D', writetype => TDOUBLE }, { name => 'MJD_MAX', type => 'D', writetype => TDOUBLE }, + # new in version 2 { name => 'RUN_TYPE', type => '16A', writetype => TSTRING }, + { name => 'FWHM_MIN', type => 'D', writetype => TDOUBLE }, + { name => 'FWHM_MAX', type => 'D', writetype => TDOUBLE }, { name => 'COMMENT ', type => '64A', writetype => TSTRING }, @@ -162,4 +169,8 @@ $req_name = $header->[0]->{value}; } +if ($email) { + $header->[$email_column_num]->{value} = $email; +} + die "no request name defined" unless defined $req_name; @@ -299,5 +310,5 @@ $$r_extver = $vals[1]; - if ($extver > 1) { + if ($$r_extver > 1) { die "number of header columns in input $nvals does not equal expected number of header words $nhead" if (@vals != @$header); @@ -323,4 +334,6 @@ if ($$r_extver == 1) { # pop off the colData arrays to account for the unused columns + pop @colData; + pop @colData; pop @colData; pop @colData; @@ -346,5 +359,5 @@ } # trim leading whitespace from comment - $comment =~ s/^\s+//; + $comment =~ s/^\s+// if $comment; if ($comment) { if ($comment =~ /#/) { Index: /tags/ipp-20130307/pstamp/scripts/pstamp_webrequest.pl =================================================================== --- /tags/ipp-20130307/pstamp/scripts/pstamp_webrequest.pl (revision 35488) +++ /tags/ipp-20130307/pstamp/scripts/pstamp_webrequest.pl (revision 35489) @@ -71,9 +71,10 @@ } -# make a request file +# make the request file in a sub directory of the current directory my $cur_dir = getcwd(); #print STDERR "cur_dir is $cur_dir\n"; +# put file in directory for the current date my $datestr = strftime "%Y/%m/%d", gmtime; my $datedir = "$cur_dir/webreq/$datestr"; Index: /tags/ipp-20130307/pstamp/scripts/pstampparse.pl =================================================================== --- /tags/ipp-20130307/pstamp/scripts/pstampparse.pl (revision 35488) +++ /tags/ipp-20130307/pstamp/scripts/pstampparse.pl (revision 35489) @@ -27,5 +27,5 @@ my $outdir; my $product; -my $label; +my $label = ""; my $save_temps; my $no_update; @@ -57,4 +57,8 @@ die "outdir is required" if !$outdir; die "product is required" if !$product; +} else { + $req_id = 0; + $outdir = "nowhere"; + $product = "dummy"; } @@ -83,32 +87,46 @@ $pstamptool .= " -dbserver $dbserver" if $dbserver; -# list_job is a deugging mode -$no_update = 1 if $mode eq "list_job"; +# If $mode is not queue_job we are using a debugging mode +# do not update the database +$no_update = 1 if $mode ne "queue_job"; my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files # -# get the data from the extension header +# Read the keywords from the extension header # my $fields_output; { - my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USER EMAIL"; + my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION EMAIL"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - # fields doesn't return zero when it succeeds - #unless ($success) { - # print STDERR @$stderr_buf; - #} + + # note fields doesn't return zero when it succeeds. $fields_output = join "", @$stdout_buf; } -my (undef, $extname, $extver, $req_name, $action, $username, $email) = split " ", $fields_output; +my (undef, $extname, $extver, $req_name, $action, $email) = split " ", $fields_output; # make sure the file contains what we are expecting -# This program shouldn't have been run if the request file is bogus. -# No need to notify the client -my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST"; -my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR) if (!$req_name); -my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2"); - +# pstamp_parser_run.pl would not have run this program unless the request file was ok +my_die("$request_file_name does not contain EXTNAME\n", $PS_EXIT_PROG_ERROR) if !$extname; +my_die("$request_file_name is not a PS1_PS_REQUEST\n", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST"; +my_die("REQ_NAME not found in $request_file_name\n", $PS_EXIT_PROG_ERROR) if (!$req_name); +my_die("wrong EXTVER $extver found in $request_file_name\n", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2"); + +if ($extver >= 2) { + # We have a version 2 file. Require that the new keywords be supplied. + my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action; + + my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW'); + + my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email; + # XXX check for "valid" $email +} else { + # for version 1 file the action is process and email is not used + $action = 'PROCESS'; + $email = 'null'; +} + +print "Request Header Keywords EXTVER: $extver REQ_NAME: $req_name ACTION: $action EMAIL: $email\n"; # check for duplicate request name @@ -161,4 +179,5 @@ # the output data store's product name my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name"; + $command .= " -set_username $email" if $email ne 'null'; $command .= " -set_outProduct $product"; $command .= " -set_label $label" if $label_changed; @@ -174,5 +193,5 @@ # -# now convert the request table to an array of metadata config docs +# now convert the request table to an array of metadatas # @@ -191,20 +210,30 @@ my $dtime_request_file = gettimeofday() - $start_request_file; print "Time to read and parse request file: $dtime_request_file\n"; - -} - -# -# Loop over rows in the request file collecting consecutive rows that have the "same images of interest" -# in the sense that their selection parameters will yield the same "Runs". -# Process the groups of rows together to reduce lookup time and to potentially make multiple -# stamps from the same ppstamp process. -# -my @rowList; +} + +my $nRows = scalar @$rows; +print "\n$nRows rows read from request file\n"; + + my $num_jobs = 0; my $imageList; my $stage; -my $need_magic; foreach my $row (@$rows) { - # santiy check the paramaters + + if ($label eq 'WEB.UP' and ($nRows > 500 or $num_jobs > 500) and $req_id and !$no_update) { + # this is a big request and it came from the upload page and doesn't have a specific label assigned + # change it to the generic one that has lower with lower priority + $label = 'WEB.BIG'; + print "\nChanging label for big WEB.UP request to $label\n"; + + my $command = "$pstamptool -updatereq -req_id $req_id -set_label $label"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR); + } + } + + # validate the paramaters if (!checkRow($row)) { # when it enconters an error checkRow adds a fake job with an appropriate error code to the database @@ -216,27 +245,5 @@ $row->{error_code} = 0; - if (scalar @rowList == 0) { - push @rowList, $row; - next; - } - - my $firstRow = $rowList[0]; - if (same_images_of_interest($firstRow, $row)) { - # add this row to the list and move on - push @rowList, $row; - next; - } - - # the images of interest for this new row doesn't match the list. - # process the list ... - $num_jobs += processRows($action, \@rowList); - - # and reset the list to contain just the new row - @rowList = ($row); -} - -# out of rows process the list -if (scalar @rowList > 0) { - $num_jobs += processRows($action, \@rowList); + $num_jobs += processRow($action, $row); } @@ -252,5 +259,7 @@ my $row = shift; + # check validity of the paramters in a request specification # If we encounter an error for a particular row add a job with the proper fault code. + # also adjust some paramterers like filter and set defaults for some others my $stage = $row->{IMG_TYPE}; @@ -284,5 +293,4 @@ } - my $component = $row->{COMPONENT}; if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) { @@ -314,4 +322,22 @@ return 0; } + + my $fwhm_min = $row->{FWHM_MIN}; + if (!defined $fwhm_min) { + $row->{FWHM_MIN} = 0; + } elsif (!validNumber($fwhm_min)) { + print STDERR "$fwhm_min is not a valid FWHM_MIN\n"; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + my $fwhm_max = $row->{FWHM_MAX}; + if (!defined $fwhm_max) { + $row->{FWHM_MAX} = 0; + } elsif (!validNumber($fwhm_max)) { + print STDERR "$fwhm_max is not a valid FWHM_MAX\n"; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + my $data_group = $row->{DATA_GROUP}; if (!defined $data_group) { @@ -334,5 +360,6 @@ my $wholefile = 0; - if (!$skycenter && $row->{CENTER_X} == 0 && $row->{CENTER_Y} == 0) { + if (($row->{WIDTH} == 0 && $row->{HEIGHT} == 0) || + (!$skycenter && $row->{CENTER_X} == 0 && $row->{CENTER_Y} == 0)) { # Secret code for returning the whole file $wholefile = 1; @@ -362,9 +389,4 @@ return 0; } - - # $mode list_uri is a debugging mode (it may used by the http interface) - # if this happens just croak - # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri"); - if ($req_type eq "bycoord") { @@ -384,4 +406,5 @@ } + return 1; } @@ -392,4 +415,65 @@ $num_jobs = 1; + return $num_jobs; +} + +sub processRow { + my $action = shift; + my $row = shift; + + my $num_jobs = 0; + + my $project = $row->{PROJECT}; + + # note: resolve_project avoids running pstamptool every time by remembering the + # last project resolved + my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); + if (!$proj_hash) { + insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PROJECT); + $num_jobs++; + return $num_jobs; + } + my $image_db = $proj_hash->{dbname}; + my $camera = $proj_hash->{camera}; + + my $req_type = $row->{REQ_TYPE}; + my $rownum = $row->{ROWNUM}; + + # Since user can get unmagicked data "by coordinate" requests can go back in time + # to dredge unusable data from the "dark days"... + if ($req_type eq 'bycoord' and $row->{IMG_TYPE} ne 'stack' and $row->{MJD_MIN} == 0) { + # ... so unless the user sets mjd_min clamp it to 2009-04-01 + # XXX: This value should live in the pstampProject table not be hardcoded here + $row->{MJD_MIN} = 54922; + } + + if (($req_type eq 'byskycell') or ($req_type eq 'bycoord')) { + # avoid error from print below if $id isn't needed + $row->{ID} = "" if !$row->{ID}; + } + + # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this + # request specification. An array reference is returned. + my $start_locate = gettimeofday(); + + print "\nCalling new_locate_images for row: $rownum\n"; + + $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose); + + my $dtime_locate = gettimeofday() - $start_locate; + print "Time to locate_images for row $rownum $dtime_locate\n"; + + my @rowList = ($row); + $num_jobs += queueJobs($action, \@rowList, $imageList); + + # if this row slipped through without a job being added add one. + if ($row->{job_num} == 0) { + print "row $row->{ROWNUM} produced no jobs\n"; + print STDERR "row $row->{ROWNUM} produced no jobs\n"; + my $error_code = $row->{error_code}; + $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code; + insertFakeJobForRow($row, ++$row->{job_num}, $error_code); + } + return $num_jobs; } @@ -401,4 +485,5 @@ if ($action eq 'LIST') { + # LIST is not allowed by caller. Can't get here. return list_targets($rowList); } @@ -420,5 +505,5 @@ } my $req_type = $row->{REQ_TYPE}; - $stage = $row->{IMG_TYPE}; + my $stage = $row->{IMG_TYPE}; my $id = $row->{ID}; my $component = $row->{COMPONENT}; @@ -436,5 +521,5 @@ my $image_db = $proj_hash->{dbname}; my $camera = $proj_hash->{camera}; - $need_magic = $proj_hash->{need_magic}; + my $need_magic = $proj_hash->{need_magic}; # Since user can get unmagicked data "by coordinate" requests can go back in time # to dredge unusable data from the "dark days"... @@ -496,5 +581,5 @@ # information required is contained there - $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component, + $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component, $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose); @@ -506,8 +591,8 @@ $row->{need_magic} = $need_magic; - $num_jobs += queueJobs($mode, \@rowList, $imageList); + $num_jobs += queueJobs($action, $rowList, $imageList); # if a row slipped through with no jobs add a faulted one - foreach my $row (@rowList) { + foreach my $row (@$rowList) { if ($row->{job_num} == 0) { print "row $row->{ROWNUM} produced no jobs\n"; @@ -528,5 +613,5 @@ my $image = shift; my $need_magic = shift; - my $mode = shift; + my $action = shift; my $rownum = $row->{ROWNUM}; @@ -544,23 +629,32 @@ my $h = $row->{HEIGHT}; my $coord_mask = $row->{COORD_MASK}; + my $wholeFile = 0; - if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) { - # Center of 0, 0 in pixel coordinates is interpreted to mean - # return the entire file - if ($x == 0 && $y == 0) { - $roi_string = "-wholefile"; - $wholeFile = 1; + # For historical reasons there are two ways to specify that the entire file be returned + # rather than a postage stamp ... + if ($w == 0 and $h == 0) { + # ... The right way: width and height both zero ... + $wholeFile = 1; + } else { + if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) { + if ($x == 0 && $y == 0) { + # ... and pixel coordinate center of 0, 0 + # I made this one up without thinking through the API clearly. + # allowing width and height to do it works much better + $wholeFile = 1; + } else { + $roi_string = "-pixcenter $x $y"; + } } else { - $roi_string = "-pixcenter $x $y"; - } + $roi_string = "-skycenter $x $y"; + } + } + + if ($wholeFile) { + $roi_string = "-wholefile"; + } elsif ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) { + $roi_string .= " -pixrange $w $h"; } else { - $roi_string = "-skycenter $x $y"; - } - if (!$wholeFile) { - if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) { - $roi_string .= " -pixrange $w $h"; - } else { $roi_string .= " -arcrange $w $h"; - } } @@ -611,9 +705,9 @@ write_params($output_base, $image); - my $newState = "run"; + my $newState = $action eq 'PROCESS' ? "run" : "parsed"; my $fault = 0; my $dep_id; - queueUpdatesIfNeeded($stage, $image, $option_mask, \$newState, \$fault, \$dep_id); + queueUpdatesIfNeeded($action, $stage, $image, $option_mask, \$newState, \$fault, \$dep_id); my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" @@ -644,9 +738,9 @@ sub queueJobs { - my $mode = shift; + my $action = shift; my $rowList = shift; my $imageList = shift; - my $firstRow = $rowList[0]; + my $firstRow = $rowList->[0]; my $stage = $firstRow->{IMG_TYPE}; my $job_type = $firstRow->{JOB_TYPE}; @@ -656,6 +750,9 @@ if ($mode eq "list_uri") { + $num_jobs = $imageList ? scalar @$imageList : 0; + print "List of $num_jobs Images selected for row: $firstRow->{ROWNUM}\n"; foreach my $image (@$imageList) { print "$image->{image}\n"; + ++$firstRow->{job_num}; } } elsif ($job_type eq "get_image") { @@ -664,5 +761,5 @@ my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1; - $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); + $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $action); } else { @@ -692,5 +789,5 @@ my $row = $rowList->[$i]; - $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode); + $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $action); } } @@ -706,5 +803,5 @@ my $stage = shift; my $need_magic = shift; - my $mode = shift; + my $action = shift; my $num_jobs = 0; @@ -753,9 +850,9 @@ write_params($output_base, $image); - my $newState = "run"; + my $newState = $action eq 'PROCESS' ? "run" : "parsed"; my $fault = 0; my $dep_id; - queueUpdatesIfNeeded($stage, $image, $option_mask, \$newState, \$fault, \$dep_id); + queueUpdatesIfNeeded($action, $stage, $image, $option_mask, \$newState, \$fault, \$dep_id); $num_jobs++; @@ -899,5 +996,5 @@ sub get_dependent { - my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_; + my ($action, $r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_; # chipRun's can be in full state if destreaking is necessary @@ -918,10 +1015,13 @@ my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $outdir"; $command .= " -need_magic" if $need_magic; - - # compute rlabel for the run. - # XXX: This bit of policy shouldn't be buried so deeply in the code - # For now use one that implies 'postage stamp server' 'update' 'request_label" - my $rlabel = "ps_ud_" . $label if $label; - $command .= " -rlabel $rlabel" if $rlabel; + $command .= ' -hold' if $action eq 'PREVIEW'; + + if ($label) { + # compute rlabel for the run. + # XXX: This bit of policy shouldn't be buried so deeply in the code + # For now use one that implies 'postage stamp server' 'update' 'request_label" + my $rlabel = "ps_ud_" . $label; + $command .= " -rlabel $rlabel"; + } if (!$no_update) { @@ -931,7 +1031,7 @@ my $fault = $error_code >> 8; print STDERR "$command failed with fault $fault\n"; - if ($fault < 10) { + if ($fault < $PSTAMP_FIRST_ERROR_CODE) { # pstamptool returns an error if an existing depenent is faulted - # Set the object to not available even if the fault < 10 + # Set the object to not available even if the fault < $PSTAMP_FIRST_ERROR_CODE # which is nominally a recoverable error in order to keep # the request from faulting (which can be very expensive if @@ -939,5 +1039,5 @@ $fault = $PSTAMP_NOT_AVAILABLE } - if ($fault >= 10) { + if ($fault >= $PSTAMP_FIRST_ERROR_CODE) { $$r_dep_id = 0; $$r_fault = $fault; @@ -967,4 +1067,5 @@ sub queueUpdatesIfNeeded { + my $action = shift; my $stage = shift; my $image = shift; @@ -973,4 +1074,6 @@ my $r_fault = shift; my $r_dep_id = shift; + + my $need_magic = 0; if ($stage ne 'raw') { @@ -1016,5 +1119,5 @@ # set up to queue an update run my $require_magic = ($need_magic or $image->{magicked}); - get_dependent(\$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, + get_dependent($action, \$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, $run_state, $stage, $image->{stage_id}, $image->{component}, $require_magic ); } @@ -1070,5 +1173,5 @@ my $fault = shift; - carp $msg; + print STDERR $msg; # we don't fault the request here pstamp_parser_run.pl handles that if necessary Index: /tags/ipp-20130307/pstamp/src/ppstampParseCamera.c =================================================================== --- /tags/ipp-20130307/pstamp/src/ppstampParseCamera.c (revision 35488) +++ /tags/ipp-20130307/pstamp/src/ppstampParseCamera.c (revision 35489) @@ -95,6 +95,4 @@ } - - // XXX: create a filerule for PPSTAMP.ASTROM pmFPAfile *astrom = pmFPAfileDefineFromArgs(&status, config, "PSWARP.ASTROM", "ASTROM"); if (!status) { @@ -103,13 +101,8 @@ } if (astrom) { - psLogMsg ("ppstamp", 3, "using supplied astrometry\n"); + psLogMsg ("ppstamp", 3, "Using supplied astrometry.\n"); } else { - psLogMsg ("ppstamp", 3, "using header astrometry\n"); + psLogMsg ("ppstamp", 3, "Using header astrometry.\n"); } - -#ifdef notyet - // add recipe options supplied on command line - psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, RECIPE_NAME); -#endif // Set up the output target @@ -138,5 +131,6 @@ } } else { - psLogMsg ("ppstamp", PS_LOG_INFO, "output sources file requested but no input supplied. Will try the supplied astrometry file.\n"); + psLogMsg ("ppstamp", PS_LOG_INFO, "Output sources file requested but no -sources supplied.\n"); + psLogMsg ("ppstamp", PS_LOG_INFO, "Will use the sources in the supplied astrometry file.\n"); } Index: /tags/ipp-20130307/pstamp/src/pstampdump.c =================================================================== --- /tags/ipp-20130307/pstamp/src/pstampdump.c (revision 35488) +++ /tags/ipp-20130307/pstamp/src/pstampdump.c (revision 35489) @@ -94,4 +94,6 @@ if (!strcmp(extname, "PS1_PS_REQUEST")) { psString extver = psMetadataLookupStr(NULL, header, "EXTVER"); + psString action = psMetadataLookupStr(NULL, header, "ACTION"); + psString email = psMetadataLookupStr(NULL, header, "EMAIL"); if (!extver) { // work around bug in MOPS request files @@ -105,5 +107,17 @@ } } - printf("%s %s %s\n", extname, extver, req_name); + if (!strcmp(extver, "1")) { + printf("%s %s %s\n", extname, extver, req_name); + } else { + if (!action) { + psErrorStackPrint(stderr, "failed to find action in fits header of: %s\n", fileName); + return PS_EXIT_DATA_ERROR; + } + if (!email) { + psErrorStackPrint(stderr, "failed to find action in fits header of: %s\n", email); + return PS_EXIT_DATA_ERROR; + } + printf("%s %s %s %s %s\n", extname, extver, req_name, action, email); + } } else if (!strcmp(extname, "PS1_PS_RESULTS")) { psS64 req_id = psMetadataLookupS64(NULL, header, "REQ_ID"); Index: /tags/ipp-20130307/pstamp/test/gpc1/chip.bycoord.norelease.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/chip.bycoord.norelease.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/chip.bycoord.norelease.txt (revision 35489) @@ -0,0 +1,7 @@ +# TEST: bycoord chip +# +# chip bycoord no survey or release date cuts +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +0 241.54949 55.4273 500 500 2 stamp 65 gpc1 null null bycoord chip null null null null i% 55674 55697 null 0 0 | Index: /tags/ipp-20130307/pstamp/test/gpc1/chip.bycoord.preview.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/chip.bycoord.preview.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/chip.bycoord.preview.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: bycoord chip +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PREVIEW null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +0 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 3PI null bycoord chip null null null null i% 0 0 null 0 0 | norelease Index: /tags/ipp-20130307/pstamp/test/gpc1/chip.bycoord.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/chip.bycoord.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/chip.bycoord.txt (revision 35489) @@ -0,0 +1,8 @@ +# TEST: bycoord chip. 3 different releases +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +0 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 3PI null bycoord chip null null null null i% 0 0 null 0 0 | norelease +0 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 3PI 3PI.PV1 bycoord chip null null null null i% 0 0 null 0 0 | GR1 +0 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 3PI 3PI.nightly bycoord chip null null null null i% 0 0 null 0 0 | nightly Index: /tags/ipp-20130307/pstamp/test/gpc1/chip.bydiff.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/chip.bydiff.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/chip.bydiff.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: bydiff chip +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null bydiff chip 16124276 null null null i 0 0 null 0 0 | Index: /tags/ipp-20130307/pstamp/test/gpc1/chip.byexp.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/chip.byexp.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/chip.byexp.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: byexp chip +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null byexp chip o6136g0055o null null null null 0 0 null 0 0 | Index: /tags/ipp-20130307/pstamp/test/gpc1/chip.byid.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/chip.byid.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/chip.byid.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: byid chip +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null byid chip 723948 null null null null 0 0 null 0 0 | Index: /tags/ipp-20130307/pstamp/test/gpc1/diff.bydiff.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/diff.bydiff.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/diff.bydiff.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: bydiff diff +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 500 500 2 stamp 65 gpc1 MD08 null bydiff diff 16124276 null null null null 0 0 null 0 0 | Index: /tags/ipp-20130307/pstamp/test/gpc1/diff.byid.inverse.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/diff.byid.inverse.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/diff.byid.inverse.txt (revision 35489) @@ -0,0 +1,8 @@ +# TEST: byid diff +# +# options mask selects inverse but first row is not bothway +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 500 500 2 stamp 1089 gpc1 MD08 null byid diff 261112 null null null null 0 0 null 0 0 | Index: /tags/ipp-20130307/pstamp/test/gpc1/diff.byid.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/diff.byid.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/diff.byid.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: byid diff +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 500 500 2 stamp 65 gpc1 MD08 null byid diff 261112 null null null null 0 0 null 0 0 | Index: /tags/ipp-20130307/pstamp/test/gpc1/extver.1.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/extver.1.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/extver.1.txt (revision 35489) @@ -0,0 +1,6 @@ +# REQ_NAME EXTVER ACTION USERNAME EMAIL +CHANGEME 1 +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX | COMMENT +0 241.54949 55.4273 1500 1500 2 stamp 2049 gpc1 byid stack 875274 null null null null 0 0 | sample v1 request tadpole galaxy i band + + Index: /tags/ipp-20130307/pstamp/test/gpc1/raw.bycoord.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/raw.bycoord.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/raw.bycoord.txt (revision 35489) @@ -0,0 +1,8 @@ +# TEST: bycoord raw. 3 different releases (should get the same results) +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +0 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 3PI null bycoord raw null null null null i% 0 0 null 0 0 | norelease +0 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 3PI 3PI.PV1 bycoord raw null null null null i% 0 0 null 0 0 | GR1 +0 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 3PI 3PI.nightly bycoord raw null null null null i% 0 0 null 0 0 | nightly Index: /tags/ipp-20130307/pstamp/test/gpc1/raw.byexp.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/raw.byexp.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/raw.byexp.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: byexp raw +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null byexp raw o6136g0055o null null null null 0 0 null 0 0 | Index: /tags/ipp-20130307/pstamp/test/gpc1/raw.byid.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/raw.byid.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/raw.byid.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: byid raw +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null byid raw 176316 null null null null 0 0 null 0 0 | Index: /tags/ipp-20130307/pstamp/test/gpc1/stack.bycoord.3pi.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/stack.bycoord.3pi.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/stack.bycoord.3pi.txt (revision 35489) @@ -0,0 +1,7 @@ +# stack bycoord from 3PI released +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 2119 gpc1 3PI null bycoord stack null null null null r 0 0 null 0 10 | tadpole galaxy r band +2 241.54949 55.4273 1500 1500 2 stamp 2119 gpc1 3PI null bycoord stack null null null null i 0 0 null 0 10 | tadpole galaxy i band Index: /tags/ipp-20130307/pstamp/test/gpc1/stack.bydiff.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/stack.bydiff.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/stack.bydiff.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: bydiff stack +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +0 241.54949 55.4273 1500 1500 2 stamp 2113 gpc1 3PI null bydiff stack 16124276 null null null i 0 0 null 0 10 | Index: /tags/ipp-20130307/pstamp/test/gpc1/stack.byid.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/stack.byid.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/stack.byid.txt (revision 35489) @@ -0,0 +1,7 @@ +# TEST: byid stack +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 2113 gpc1 3PI null byid stack 1422246 null null null r 0 0 null 0 10 | tadpole galaxy r band +2 241.54949 55.4273 1500 1500 2 stamp 2113 gpc1 3PI null byid stack 1422346 null null null i 0 0 null 0 10 | tadpole galaxy i band Index: /tags/ipp-20130307/pstamp/test/gpc1/stack.byskycell.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/stack.byskycell.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/stack.byskycell.txt (revision 35489) @@ -0,0 +1,7 @@ +# TEST: byid stack +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 2113 gpc1 3PI null byskycell stack null RINGS.V3 skycell.2386.085 null r 0 0 null 0 10 | tadpole galaxy r band +2 241.54949 55.4273 1500 1500 2 stamp 2113 gpc1 3PI null byskycell stack null RINGS.V3 skycell.2386.085 null i 0 0 null 0 10 | tadpole galaxy i band Index: /tags/ipp-20130307/pstamp/test/gpc1/stack.md08.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/stack.md08.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/stack.md08.txt (revision 35489) @@ -0,0 +1,10 @@ +# stack bycoord from MD08.refstack +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 2119 gpc1 null null bycoord stack null null null MD08.refstack.20130401 g 0 0 null 0 0 | tadpole galaxy g band +2 241.54949 55.4273 1500 1500 2 stamp 2119 gpc1 null null bycoord stack null null null MD08.refstack.20130401 r 0 0 null 0 0 | tadpole galaxy r band +3 241.54949 55.4273 1500 1500 2 stamp 2119 gpc1 null null bycoord stack null null null MD08.refstack.20130401 i 0 0 null 0 0 | tadpole galaxy i band +4 241.54949 55.4273 1500 1500 2 stamp 2119 gpc1 null null bycoord stack null null null MD08.refstack.20130401 z 0 0 null 0 0 | tadpole galaxy z band +5 241.54949 55.4273 1500 1500 2 stamp 2119 gpc1 null null bycoord stack null null null MD08.refstack.20130401 y 0 0 null 0 0 | tadpole galaxy y band Index: /tags/ipp-20130307/pstamp/test/gpc1/test_qub_ps_request_20130426_225546.tbl =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/test_qub_ps_request_20130426_225546.tbl (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/test_qub_ps_request_20130426_225546.tbl (revision 35489) @@ -0,0 +1,17 @@ +SIMPLE = T / file does conform to FITS standard BITPIX = 8 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / array data type NAXIS = 2 / number of array dimensions NAXIS1 = 412 / length of dimension 1 NAXIS2 = 63 / length of dimension 2 PCOUNT = 0 / number of group parameters GCOUNT = 1 / number of groups TFIELDS = 19 / number of table fields TTYPE1 = 'ROWNUM ' TFORM1 = 'J ' TTYPE2 = 'CENTER_X' TFORM2 = 'D ' TTYPE3 = 'CENTER_Y' TFORM3 = 'D ' TTYPE4 = 'WIDTH ' TFORM4 = 'D ' TTYPE5 = 'HEIGHT ' TFORM5 = 'D ' TTYPE6 = 'COORD_MASK' TFORM6 = 'J ' TTYPE7 = 'JOB_TYPE' TFORM7 = '16A ' TTYPE8 = 'OPTION_MASK' TFORM8 = 'J ' TTYPE9 = 'PROJECT ' TFORM9 = '16A ' TTYPE10 = 'REQ_TYPE' TFORM10 = '16A ' TTYPE11 = 'IMG_TYPE' TFORM11 = '16A ' TTYPE12 = 'ID ' TFORM12 = '16A ' TTYPE13 = 'TESS_ID ' TFORM13 = '64A ' TTYPE14 = 'COMPONENT' TFORM14 = '64A ' TTYPE15 = 'LABEL ' TFORM15 = '64A ' TTYPE16 = 'REQFILT ' TFORM16 = '16A ' TTYPE17 = 'MJD_MIN ' TFORM17 = 'D ' TTYPE18 = 'MJD_MAX ' TFORM18 = 'D ' TTYPE19 = 'COMMENT ' TFORM19 = '64A ' EXTNAME = 'PS1_PS_REQUEST' / name of this binary table extension REQ_NAME= 'test_qub_ps_request_20130426_225546' / Postage Stamp request name EXTVER = '1 ' / Extension version HISTORY File modified by user 'bills' with fv on 2013-04-26T13:24:49 END @jº7³†÷@JE èŠã@rÀ@rÀstampgpc1byiddiff423090MD07skycell.034nullnull1141516631523220800_56402.560_25666082_2_diff@jº7³†÷@JE èŠã@rÀ@rÀstampgpc1byidstack1964771MD07skycell.034nullnull1141516631523220800_56402.560_25666082_2_ref@jº7³†÷@JE èŠã@rÀ@rÀstampgpc1byidstack2211920MD07skycell.034nullnull1141516631523220800_56402.560_25666082_2_target@jº7œ©@JDÿ7‰ü§@rÀ@rÀstampgpc1byiddiff427003MD07skycell.034nullnull1141516631523220800_56408.416_25881073_81_diff@jº7œ©@JDÿ7‰ü§@rÀ@rÀstampgpc1byidstack1964705MD07skycell.034nullnull1141516631523220800_56408.416_25881073_81_ref@jº7œ©@JDÿ7‰ü§@rÀ@rÀstampgpc1byidstack2220425MD07skycell.034nullnull1141516631523220800_56408.416_25881073_81_target@jº­³T\ @JDÕ › +@rÀ@rÀstampgpc1byiddiff423090MD07skycell.034nullnull1141520081523215900_56402.560_25666082_380_diff@jº­³T\ @JDÕ › +@rÀ@rÀstampgpc1byidstack1964771MD07skycell.034nullnull1141520081523215900_56402.560_25666082_380_ref @jº­³T\ @JDÕ › +@rÀ@rÀstampgpc1byidstack2211920MD07skycell.034nullnull1141520081523215900_56402.560_25666082_380_target +@jº¯+ė@JD֚ª*ˆ@rÀ@rÀstampgpc1byiddiff427003MD07skycell.034nullnull1141520081523215900_56408.416_25881073_19_diff +@jº¯+ė@JD֚ª*ˆ@rÀ@rÀstampgpc1byidstack1964705MD07skycell.034nullnull1141520081523215900_56408.416_25881073_19_ref +@jº¯+ė@JD֚ª*ˆ@rÀ@rÀstampgpc1byidstack2220425MD07skycell.034nullnull1141520081523215900_56408.416_25881073_19_target +@jÀR<`Û@Jdåé +sÁ@rÀ@rÀstampgpc1byiddiff423090MD07skycell.034nullnull1141600911524717700_56402.560_25666082_300_diff@jÀR<`Û@Jdåé +sÁ@rÀ@rÀstampgpc1byidstack1964771MD07skycell.034nullnull1141600911524717700_56402.560_25666082_300_ref@jÀR<`Û@Jdåé +sÁ@rÀ@rÀstampgpc1byidstack2211920MD07skycell.034nullnull1141600911524717700_56402.560_25666082_300_target@jÀ *N‘6@Jdâox@rÀ@rÀstampgpc1byiddiff427003MD07skycell.034nullnull1141600911524717700_56408.416_25881073_101_diff@jÀ *N‘6@Jdâox@rÀ@rÀstampgpc1byidstack1964705MD07skycell.034nullnull1141600911524717700_56408.416_25881073_101_ref@jÀ *N‘6@Jdâox@rÀ@rÀstampgpc1byidstack2220425MD07skycell.034nullnull1141600911524717700_56408.416_25881073_101_target@j§ÐÅ9™{@J?å@òÍ­@rÀ@rÀstampgpc1byiddiff416773MD07skycell.035nullnull1141258611522957000_56392.509_25353215_4_diff@j§ÐÅ9™{@J?å@òÍ­@rÀ@rÀstampgpc1byidstack1964772MD07skycell.035nullnull1141258611522957000_56392.509_25353215_4_ref@j§ÐÅ9™{@J?å@òÍ­@rÀ@rÀstampgpc1byidstack2201969MD07skycell.035nullnull1141258611522957000_56392.509_25353215_4_target@j§ÐÈP@J?ãZ”â/@rÀ@rÀstampgpc1byiddiff427003MD07skycell.035nullnull1141258611522957000_56408.416_25881074_4_diff@j§ÐÈP@J?ãZ”â/@rÀ@rÀstampgpc1byidstack1964706MD07skycell.035nullnull1141258611522957000_56408.416_25881074_4_ref@j§ÐÈP@J?ãZ”â/@rÀ@rÀstampgpc1byidstack2220426MD07skycell.035nullnull1141258611522957000_56408.416_25881074_4_target@j§Ïù]ǘ@J?çx_ó@rÀ@rÀstampgpc1byiddiff423090MD07skycell.035nullnull1141258611522957000_56402.560_25666083_35_diff@j§Ïù]ǘ@J?çx_ó@rÀ@rÀstampgpc1byidstack1964772MD07skycell.035nullnull1141258611522957000_56402.560_25666083_35_ref@j§Ïù]ǘ@J?çx_ó@rÀ@rÀstampgpc1byidstack2211921MD07skycell.035nullnull1141258611522957000_56402.560_25666083_35_target +@j§Ñ|[w(@J?ãBV˜@rÀ@rÀstampgpc1byiddiff419665MD07skycell.035nullnull1141258611522957000_56397.431_25498242_17_diff +@j§Ñ|[w(@J?ãBV˜@rÀ@rÀstampgpc1byidstack1964772MD07skycell.035nullnull1141258611522957000_56397.431_25498242_17_ref +@j§Ñ|[w(@J?ãBV˜@rÀ@rÀstampgpc1byidstack2204406MD07skycell.035nullnull1141258611522957000_56397.431_25498242_17_target@j›aô¶Æ!@J[ì7Íö[@rÀ@rÀstampgpc1byiddiff419665MD07skycell.035nullnull1141125361524305300_56397.431_25498242_49_diff @j›aô¶Æ!@J[ì7Íö[@rÀ@rÀstampgpc1byidstack1964772MD07skycell.035nullnull1141125361524305300_56397.431_25498242_49_ref!@j›aô¶Æ!@J[ì7Íö[@rÀ@rÀstampgpc1byidstack2204406MD07skycell.035nullnull1141125361524305300_56397.431_25498242_49_target"@j›aÑ7Ù8@J[ëx†y|@rÀ@rÀstampgpc1byiddiff427003MD07skycell.035nullnull1141125361524305300_56408.416_25881074_25_diff#@j›aÑ7Ù8@J[ëx†y|@rÀ@rÀstampgpc1byidstack1964706MD07skycell.035nullnull1141125361524305300_56408.416_25881074_25_ref$@j›aÑ7Ù8@J[ëx†y|@rÀ@rÀstampgpc1byidstack2220426MD07skycell.035nullnull1141125361524305300_56408.416_25881074_25_target%@j›`ÿ$SŸ@J[éÑV{@rÀ@rÀstampgpc1byiddiff424459MD07skycell.035nullnull1141125361524305300_56404.546_25741368_14_diff&@j›`ÿ$SŸ@J[éÑV{@rÀ@rÀstampgpc1byidstack1964772MD07skycell.035nullnull1141125361524305300_56404.546_25741368_14_ref'@j›`ÿ$SŸ@J[éÑV{@rÀ@rÀstampgpc1byidstack2212743MD07skycell.035nullnull1141125361524305300_56404.546_25741368_14_target(@j›`ÞJc•@J[éD+£=@rÀ@rÀstampgpc1byiddiff423090MD07skycell.035nullnull1141125361524305300_56402.560_25666083_34_diff)@j›`ÞJc•@J[éD+£=@rÀ@rÀstampgpc1byidstack1964772MD07skycell.035nullnull1141125361524305300_56402.560_25666083_34_ref*@j›`ÞJc•@J[éD+£=@rÀ@rÀstampgpc1byidstack2211921MD07skycell.035nullnull1141125361524305300_56402.560_25666083_34_target+@jq^PTxá@JX±óvI +@rÀ@rÀstampgpc1byiddiff410657MD07skycell.037nullnull1140610261524134500_56383.565_25053115_30_diff,@jq^PTxá@JX±óvI +@rÀ@rÀstampgpc1byidstack1964774MD07skycell.037nullnull1140610261524134500_56383.565_25053115_30_ref-@jq^PTxá@JX±óvI +@rÀ@rÀstampgpc1byidstack2196005MD07skycell.037nullnull1140610261524134500_56383.565_25053115_30_target.@jq_d` +k@JX³(:‡;@rÀ@rÀstampgpc1byiddiff427003MD07skycell.037nullnull1140610261524134500_56408.416_25881076_37_diff/@jq_d` +k@JX³(:‡;@rÀ@rÀstampgpc1byidstack1964708MD07skycell.037nullnull1140610261524134500_56408.416_25881076_37_ref0@jq_d` +k@JX³(:‡;@rÀ@rÀstampgpc1byidstack2220428MD07skycell.037nullnull1140610261524134500_56408.416_25881076_37_target1@jq]Œi(Š@JX®k<Ú@rÀ@rÀstampgpc1byiddiff424459MD07skycell.037nullnull1140610261524134500_56404.546_25741370_9_diff2@jq]Œi(Š@JX®k<Ú@rÀ@rÀstampgpc1byidstack1964774MD07skycell.037nullnull1140610261524134500_56404.546_25741370_9_ref3@jq]Œi(Š@JX®k<Ú@rÀ@rÀstampgpc1byidstack2212745MD07skycell.037nullnull1140610261524134500_56404.546_25741370_9_target4@jq^·Ÿ]@JX°k‚¶@rÀ@rÀstampgpc1byiddiff423090MD07skycell.037nullnull1140610261524134500_56402.560_25666085_42_diff5@jq^·Ÿ]@JX°k‚¶@rÀ@rÀstampgpc1byidstack1964774MD07skycell.037nullnull1140610261524134500_56402.560_25666085_42_ref6@jq^·Ÿ]@JX°k‚¶@rÀ@rÀstampgpc1byidstack2211923MD07skycell.037nullnull1140610261524134500_56402.560_25666085_42_target7@jq^÷ÉT(@JXŽí€*H@rÀ@rÀstampgpc1byiddiff419665MD07skycell.037nullnull1140610261524134500_56397.431_25498244_45_diff8@jq^÷ÉT(@JXŽí€*H@rÀ@rÀstampgpc1byidstack1964774MD07skycell.037nullnull1140610261524134500_56397.431_25498244_45_ref9@jq^÷ÉT(@JXŽí€*H@rÀ@rÀstampgpc1byidstack2204408MD07skycell.037nullnull1140610261524134500_56397.431_25498244_45_target:@já<ˆ—;@Ju£œIߒ@rÀ@rÀstampgpc1byiddiff422486MD07skycell.042nullnull1142009271525508600_56401.503_25632812_13_diff;@já<ˆ—;@Ju£œIߒ@rÀ@rÀstampgpc1byidstack1964910MD07skycell.042nullnull1142009271525508600_56401.503_25632812_13_ref<@já<ˆ—;@Ju£œIߒ@rÀ@rÀstampgpc1byidstack2210439MD07skycell.042nullnull1142009271525508600_56401.503_25632812_13_target=@já;µÐ@Ju£ÐÞ@rÀ@rÀstampgpc1byiddiff427003MD07skycell.042nullnull1142009271525508600_56408.416_25881079_7_diff>@já;µÐ@Ju£ÐÞ@rÀ@rÀstampgpc1byidstack1964712MD07skycell.042nullnull1142009271525508600_56408.416_25881079_7_ref?@já;µÐ@Ju£ÐÞ@rÀ@rÀstampgpc1byidstack2220432MD07skycell.042nullnull1142009271525508600_56408.416_25881079_7_target Index: /tags/ipp-20130307/pstamp/test/gpc1/tests_todo =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/tests_todo (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/tests_todo (revision 35489) @@ -0,0 +1,10 @@ +each img_type +each req_type + make a request using each of the modes + add qualifiers + test options + test using data from various epochs + test coordinates that are outside of the selected image to insure error code is ok +both request versions +a few samples were done + Index: /tags/ipp-20130307/pstamp/test/gpc1/warp.bycoord.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/warp.bycoord.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/warp.bycoord.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: bycoord warp +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null bycoord warp null null null null i 56136 56137 null 0 10 | Index: /tags/ipp-20130307/pstamp/test/gpc1/warp.bycoord.wrongskycell.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/warp.bycoord.wrongskycell.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/warp.bycoord.wrongskycell.txt (revision 35489) @@ -0,0 +1,7 @@ +# TEST: bycoord warp coordinates do not match skycell_id +# should get no overlap error +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null bycoord warp null null skycell.065 null i 56136 56137 null 0 10 | Index: /tags/ipp-20130307/pstamp/test/gpc1/warp.bydiff.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/warp.bydiff.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/warp.bydiff.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: bydiff stack +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null bydiff warp 16124276 null null null i 0 0 null 0 10 | Index: /tags/ipp-20130307/pstamp/test/gpc1/warp.byexp.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/warp.byexp.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/warp.byexp.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: byid stack +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null byid warp 471197 null null null i 0 0 null 0 10 | Index: /tags/ipp-20130307/pstamp/test/gpc1/warp.byid.noverlap.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/warp.byid.noverlap.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/warp.byid.noverlap.txt (revision 35489) @@ -0,0 +1,9 @@ +# TEST: byid warp no overlap. Coordinates are in skycell.065 not the +# requested skycell.066. +# Note that this causes a job to be queued even though the result is going to +# be PSTAMP_NO_OVERLAP +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 242.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null byid warp 471197 null skycell.066 null i 0 0 null 0 0 | Index: /tags/ipp-20130307/pstamp/test/gpc1/warp.byid.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/warp.byid.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/warp.byid.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: byexp warp +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null byexp warp o6136g0055o null null null i 0 0 null 0 10 | Index: /tags/ipp-20130307/pstamp/test/gpc1/warp.byskycell.txt =================================================================== --- /tags/ipp-20130307/pstamp/test/gpc1/warp.byskycell.txt (revision 35489) +++ /tags/ipp-20130307/pstamp/test/gpc1/warp.byskycell.txt (revision 35489) @@ -0,0 +1,6 @@ +# TEST: byskycell warp +# +# REQ_NAME EXTVER ACTION EMAIL +CHANGEME 2 PROCESS null +# ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT +1 241.54949 55.4273 1500 1500 2 stamp 65 gpc1 MD08 null byskycell warp null MD08.V3 skycell.066 null i 56136 56137 null 0 10 | tadpole galaxy i band Index: /tags/ipp-20130307/pstamp/test/maketestreq =================================================================== --- /tags/ipp-20130307/pstamp/test/maketestreq (revision 35489) +++ /tags/ipp-20130307/pstamp/test/maketestreq (revision 35489) @@ -0,0 +1,143 @@ +#!/bin/env perl +### +### maketest +### +### Program to take a text format postage stamp request specification +### and generate a postage stamp request fits file and optionally submit it for processing +### + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions ) ; # :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( :standard ); +use PS::IPP::PStamp::RequestFile qw( :standard ); +use PS::IPP::PStamp::Job qw( :standard ); +use File::Temp qw(tempfile); +use File::Basename qw(basename); +use IPC::Cmd 0.36 qw( can_run run ); +use Carp; +use POSIX; + +my $missing_tools; +my $pstamp_request_file = can_run('pstamp_request_file') or (warn "Can't find required program pstamp_request_file" and $missing_tools = 1); +my $pstamptool = can_run('pstamptool') or (warn "Can't find required program pstamptool" and $missing_tools = 1); +my $pstampdump = can_run('pstampdump') or (warn "Can't find required program pstampdump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); +} + +# XXX: get these from site.config +$pstamptool .= " -dbserver ippc19 -dbname ippRequestServer"; + + +my ($input, $submit, $listfile, $delete, $simple, $tag_req_name, $req_name, $email); +my ($save_temps, $verbose); + +my $req_name_base = 'test.'; +my $label = 'TEST'; +my $outdir = $ENV{PWD}; + +GetOptions( + 'input|i=s' => \$input, + 'outdir=s' => \$outdir, + 'submit|s' => \$submit, + 'req_name=s' => \$req_name, + 'req_name_base=s' => \$req_name_base, + 'tag_req_name' => \$tag_req_name, + 'label=s' => \$label, + 'email=s' => \$email, + + 'listfile|l' => \$listfile, + 'simple' => \$simple, + 'delete' => \$delete, # delete request file (after listing perhaps disabled if $submit + + 'verbose|v' => \$verbose, + 'save-temps' => \$save_temps, +) or pod2usage(2); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( -msg => "Required options: --input") + unless (defined $input) ; +pod2usage( -msg => "Required options: --req_name or --req_name_base") + unless (defined $req_name or defined $req_name_base); + +die ("Input file $input not found\n") unless (-e $input); +die ("Input file $input not readable\n") unless (-r $input); + +# if request name was not supplied make one +if (!$req_name) { + if ($tag_req_name) { + # use the input file name with .txt removed as the tag + my $tag = $input; + $tag =~ s/\.txt$//; + $req_name_base = "$req_name_base$tag."; + } + my $datestr = strftime "%Y%m%dT%H%M%S", gmtime; + $req_name .= $req_name_base . $datestr; +} + +my $table_extension = 'tbl'; + +if (! ($outdir =~ /^\//) ) { + # turn relative path into absolute + $outdir = $ENV{PWD} . "/$outdir"; +} + +my $request_file_name = "$outdir/$req_name.$table_extension"; + +# build the request fits table +{ + my $command = "$pstamp_request_file --input $input --output $request_file_name"; + $command .= " --req_name $req_name" if $req_name; + $command .= " --email $email" if $email; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + exit $error_code >> 8; + } +} +die ("cannot find request file $request_file_name\n") unless -e $request_file_name; + +# optionally use pstampdump to show the contents of the request file +if ($listfile) { + my $command = "$pstampdump -header $request_file_name"; + $command .= " -simple" if $simple; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + exit $error_code >> 8; + } + print join "", @$stdout_buf; +} + +# optionally submit the request by adding it to the database. +# Note that once the request has been processed the request file is copied to the postage stamp working +# directories so these files may be deleted. +if ($submit) { + my $command = "$pstamptool -addreq -uri $request_file_name -label $label"; + $command .= " -username $email" if $email; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + exit $error_code >> 8; + } + my $output = join "", @$stdout_buf; + print "RequestFile $request_file_name submitted. req_id $output"; +} else { + # we aren't submitting the file that we just built + # optionally delete it + if ($delete) { + unlink $request_file_name or die "failed to delete request file $request_file_name\n"; + } else { + print "RequestFile $request_file_name\n"; + } +} + +exit 0;