Index: branches/cnb_branch_20081104/pstamp/scripts/pstamp_finish.pl =================================================================== --- branches/cnb_branch_20081011/pstamp/scripts/pstamp_finish.pl (revision 20067) +++ branches/cnb_branch_20081104/pstamp/scripts/pstamp_finish.pl (revision 20530) @@ -74,12 +74,15 @@ print STDERR "output fileset directory $out_dir does not exist\n"; - # XXX TODO: if this fails fault the request so we pstamp_finish - - mkdir $out_dir or die "cannot create output directory $out_dir"; + if (!mkdir $out_dir) { + print STDERR "cannot create output directory $out_dir"; + stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR, $dbname); + } + } elsif (! -d $out_dir ) { # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the # request - die "output fileset directory $out_dir exists but is not a directory"; + print STDERR "output fileset directory $out_dir exists but is not a directory"; + stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR, $dbname); } @@ -87,5 +90,4 @@ print STDERR "request file $req_file is missing\n"; stop_request($req_id, $PS_EXIT_CONFIG_ERROR, $dbname); - exit $PS_EXIT_CONFIG_ERROR; } @@ -98,5 +100,4 @@ print STDERR "failed to read request_file $req_file" ; stop_request($req_id, $PS_EXIT_CONFIG_ERROR, $dbname); - exit $PS_EXIT_CONFIG_ERROR; } @@ -109,4 +110,6 @@ my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps); print $rlf "results.fits|||table|\n"; + + # XXX: this file is in the request's workdir not the out_dir my $err_file = "$out_dir/parse_error.txt"; if (-e $err_file ) { @@ -137,10 +140,10 @@ # including those that produced no jobs. # for now add an entry for rownum 1 and a phony error code. - # we've included parse_results.txt to the fileset if it exists + # we've included parse_error.txt to the fileset if it exists # - my $rownum = 1; + my $rownum = 0; my $fault = 42; # get a real error code print $tdf "$rownum|$fault|0|0|"; - print $tdf "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|\n"; + print $tdf "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|\n"; } else { my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); @@ -159,10 +162,10 @@ my $exp_id = $job->{exp_id}; - # XXX: get the image_db from a table in the database, or maybe save it in the job - - # unless $exp_id is null (e.g. stack images) get the metadata for this exposure my ($row, $req_info, $project) = get_request_info($rows, $rownum); - my $proj_hash = resolve_project($project); + my $proj_hash = resolve_project($ipprc, $project, $dbname); my $image_db = $proj_hash->{dbname}; + + # get the metadata for the exposure (if any i.e. stack) + # returns an appropriate string if !$exp_id my $exp_info = get_exposure_info($image_db, $exp_id); @@ -182,9 +185,7 @@ # ra_deg and dec_deg are the coordinates of center of the stamp - # - # XXX we need to get the center coordinate of the stamp. - # if request used -skycenter then we have it in the request info - + # XXX: parse the stamp header to find it print $tdf "0.0|0.0|"; + print $tdf "$exp_info|"; print $tdf "$req_info|"; @@ -233,4 +234,5 @@ print STDERR "Unable to perform $command error code: $error_code\n"; $request_fault = $error_code >> 8; + # fall through to stop the request } } @@ -260,4 +262,5 @@ die("Unable to perform $command error code: $error_code"); } + exit $fault; } Index: branches/cnb_branch_20081104/pstamp/scripts/pstamp_get_image_job.pl =================================================================== --- branches/cnb_branch_20081011/pstamp/scripts/pstamp_get_image_job.pl (revision 20067) +++ branches/cnb_branch_20081104/pstamp/scripts/pstamp_get_image_job.pl (revision 20530) @@ -106,4 +106,5 @@ my $pathname = shift @fields; my $filetype = shift @fields; + my $class_id = shift @fields; ($pathname , my $filename) = resolvepath($pathname); @@ -121,5 +122,5 @@ # XXX is pstamp always the right file type, if not where can we get the right one? - print REGLIST file_registration_line($filename, $dest_path, $filetype); + print REGLIST file_registration_line($filename, $dest_path, $filetype, $class_id); foreach my $f (@fields) { @@ -136,4 +137,8 @@ my $pathname = $_[0]; + # use the basename of the unresolved file as the name of the file + # in order to avoid nebulous name mangled paths in the datastore + my $file = basename($pathname); + my $slash = index($pathname, "/"); if ($slash != 0) { @@ -141,8 +146,7 @@ $ipprc = PS::IPP::Config->new(); } - $pathname = $ipprc->convert_filename_absolute($pathname); + $pathname = $ipprc->file_resolve($pathname); } - my $file = basename($pathname); return ($pathname, $file); @@ -154,4 +158,7 @@ my $path = shift; my $filetype = shift; + my $chipname = shift; + $chipname = $chipname ? "$chipname|" : ""; + if (-e $path) { my @finfo = stat($path); @@ -160,5 +167,5 @@ my $md5sum = file_md5_hex($path); - return "$filename|$bytes|$md5sum|$filetype|"; + return "$filename|$bytes|$md5sum|$filetype|$chipname"; } else { die "$filename not found at $path"; Index: branches/cnb_branch_20081104/pstamp/scripts/pstamp_job_run.pl =================================================================== --- branches/cnb_branch_20081011/pstamp/scripts/pstamp_job_run.pl (revision 20067) +++ branches/cnb_branch_20081104/pstamp/scripts/pstamp_job_run.pl (revision 20530) @@ -13,11 +13,5 @@ use File::Basename; use Digest::MD5::File qw( file_md5_hex ); - -# values for OPTION_MASK -# XXX put these in a module -use constant PSTAMP_SELECT_IMAGE => 1; -use constant PSTAMP_SELECT_MASK => 2; -use constant PSTAMP_SELECT_WEIGHT => 4; - +use PStamp::RequestFile qw( :standard ); my $verbose; @@ -124,11 +118,24 @@ open F, ">$reglist" or die "can't open $reglist for output"; - # we are assuming an interface with ppstamp here. + # figure out what output images we should expect + + # Note: we are assuming the contents of the filerules here. my @extensions = ( "fits", "mk.fits", "wt.fits"); - # XXX TODO: add mask of expected file types to pstampJob so we know what to expect - - my $output_mask = PSTAMP_SELECT_IMAGE; - my $m = 1; # XXX we're getting a bit intimate with the bit field definitions here. do better use a hash + # we always require an image + my $output_mask = $PSTAMP_SELECT_IMAGE; + + # we search the argString for -mask and -weight. + # searching the arg string allows us to avoid adding a column in pstampJob + # XXX yeah but does this make us susceptible to user input fouling us up? + # For example, what if somebody sets the output base to file-mask-weight + # TODO: clean this up + $output_mask |= $PSTAMP_SELECT_MASK if ($argString =~ /-mask/); + $output_mask |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-weight/); + + # XXX we're getting a bit intimate with the bit field definitions here. + # do better use a hash + + my $m = 1; foreach my $extension (@extensions) { my $do_this_one = $m & $output_mask; @@ -141,5 +148,5 @@ my $path = "${outputBase}.${extension}"; - # XXX is pstamp always the right file type, if not where can we get the right one? + # XXX is pstamp always the right file type, if not where do we get the right one? print F file_registration_line($filename, $path, "pstamp") . "\n"; } @@ -182,5 +189,7 @@ exit $jobStatus; +# create a string to be passed as input to dsreg when registering this file in a fileset # XXX move this to a module so it can be shared + sub file_registration_line { my $filename = shift; Index: branches/cnb_branch_20081104/pstamp/scripts/pstamp_results_file.pl =================================================================== --- branches/cnb_branch_20081011/pstamp/scripts/pstamp_results_file.pl (revision 20067) +++ branches/cnb_branch_20081104/pstamp/scripts/pstamp_results_file.pl (revision 20530) @@ -91,5 +91,4 @@ # output parameters -# { name => 'STAMP_NAME', type => '16A', writetype => TSTRING }, { name => 'OPTION_MASK',type => 'J', writetype => TULONG }, Index: branches/cnb_branch_20081104/pstamp/scripts/pstamp_revert_request.pl =================================================================== --- branches/cnb_branch_20081011/pstamp/scripts/pstamp_revert_request.pl (revision 20067) +++ branches/cnb_branch_20081104/pstamp/scripts/pstamp_revert_request.pl (revision 20530) @@ -41,4 +41,6 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage ( -msg => "--req_id is required", -exitval => 2 ) if !$req_id; my $missing_tools; Index: branches/cnb_branch_20081104/pstamp/scripts/pstampparse.pl =================================================================== --- branches/cnb_branch_20081011/pstamp/scripts/pstampparse.pl (revision 20067) +++ branches/cnb_branch_20081104/pstamp/scripts/pstampparse.pl (revision 20530) @@ -123,4 +123,5 @@ } +my $num_jobs = 0; foreach my $row (@$rows) { my $rownum = $row->{ROWNUM}; @@ -131,5 +132,4 @@ my $id = $row->{ID}; my $class_id = $row->{CLASS_ID}; - my $stamp_name = $row->{STAMP_NAME}; my $filter = $row->{REQFILT}; my $mjd_min = $row->{MJD_MIN}; @@ -142,5 +142,4 @@ my $option_mask= $row->{OPTION_MASK}; - $stamp_name = "" if $stamp_name eq "null"; $class_id = "" if ($class_id eq "null" or $class_id eq "all"); @@ -192,4 +191,8 @@ $x, $y, $mjd_min, $mjd_max, $filter); + if (!$images) { + print STDERR "no matching images found for row $rownum\n"; + next; + } if ($mode eq "list_uri") { foreach my $image (@$images) { @@ -201,11 +204,23 @@ my $listfile = "$out_dir/filelist"; + + # map our img_type to the Data Store file types. + my %filelist_img_types = ( "raw" => "chip", + "chip" => "chipproc", + "warp" => "warp", + "stack"=>"stack", + "diff" => "diff"); + + my $filelist_img_type = $filelist_img_types{$img_type}; + open LISTFILE, ">$listfile" or die "failed to open file list: $listfile while parsing get_image request $req_id"; foreach my $image (@$images) { - print LISTFILE "$image->{image}|$img_type\n"; + my $class_id = $image->{class_id} ? $image->{class_id} : ""; + print LISTFILE "$image->{image}|$filelist_img_type|$class_id|\n"; } close LISTFILE; + $num_jobs++; my $command = "$pstamptool -addjob -req_id $req_id -job_type get_image" . " -uri $listfile -outputBase $out_dir -rownum $rownum"; @@ -234,11 +249,9 @@ my $uri = $image->{image}; my $exp_id = $image->{exp_id}; + $job_num++; my $output_base = "$out_dir/${rownum}_${job_num}"; - - # By commenting out the next line, I hereby drop the use of STAMP_NAME - # $output_base .= "_${stamp_name}" if $stamp_name; # add astrometry file for raw and chip images if one is available @@ -247,8 +260,8 @@ } - if ($option_mask & $PSTAMP_SELECT_MASK) { + if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { $args .= " -mask $image->{mask}"; } - if ($option_mask & $PSTAMP_SELECT_WEIGHT) { + if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { $args .= " -weight $image->{weight}"; } @@ -265,4 +278,5 @@ my $state = "run"; + $num_jobs++; my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type" . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum" @@ -291,3 +305,13 @@ } -exit 0; +if ($mode eq "queue_jobs") { + if ($num_jobs) { + exit 0; + } else { + # no jobs queued return error to get the request stopped + # TODO: need to define meaningful error codes + exit $PS_EXIT_UNKNOWN_ERROR; + } +} else { + exit 0; +}